{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max_steps:  1600\n",
      "device:  cuda:0\n",
      "state dim:  24\n",
      "action dim:  Box(4,)\n",
      "leraning rate:  0.0005\n",
      "updates_per_step:  4\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import torch\n",
    "from sac_agent import soft_actor_critic_agent\n",
    "from replay_memory import ReplayMemory\n",
    "\n",
    "seed=0  \n",
    "env = gym.make('BipedalWalker-v2')\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "env.seed(seed)\n",
    "max_steps = env._max_episode_steps\n",
    "print('max_steps: ', max_steps)\n",
    "\n",
    "batch_size=256\n",
    "\n",
    "LEARNING_RATE=0.0005 # lr = 0.0001 for BipedalWalker-SAC\n",
    "eval=True  ## \n",
    "start_steps=10000 ## Steps sampling random actions\n",
    "replay_size=1000000 ## size of replay buffer\n",
    "updates_per_step=4\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# alpha=0.2  # relative importance of the entropy\n",
    "# gamma=0.99  # discount factor \n",
    "# tau=0.005  # target smoothing coefficient(τ)\n",
    "\n",
    "agent = soft_actor_critic_agent(env.observation_space.shape[0], env.action_space, \\\n",
    "        device=device, hidden_size=256, lr=LEARNING_RATE, gamma=0.99, tau=0.005, alpha=0.2)\n",
    "\n",
    "memory = ReplayMemory(replay_size)\n",
    "\n",
    "print('device: ', device)\n",
    "print('state dim: ', env.observation_space.shape[0])\n",
    "print('action dim: ', env.action_space)\n",
    "print('leraning rate: ', LEARNING_RATE)\n",
    "print('updates_per_step: ', updates_per_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save(agent, directory, filename, suffix):\n",
    "    torch.save(agent.policy.state_dict(), '%s/%s_actor_%s.pth' % (directory, filename, suffix))\n",
    "    torch.save(agent.critic.state_dict(), '%s/%s_critic_%s.pth' % (directory, filename, suffix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 0, Total Steps: 90, Ep.Steps: 90, Score: -98.13, Avg.Score: -98.13, Time: 00:00:00\n",
      "Ep.: 1, Total Steps: 151, Ep.Steps: 61, Score: -114.96, Avg.Score: -106.55, Time: 00:00:00\n",
      "Ep.: 2, Total Steps: 229, Ep.Steps: 78, Score: -100.64, Avg.Score: -104.58, Time: 00:00:00\n",
      "Ep.: 3, Total Steps: 1829, Ep.Steps: 1600, Score: -87.41, Avg.Score: -100.29, Time: 00:01:16\n",
      "Ep.: 4, Total Steps: 1875, Ep.Steps: 46, Score: -112.80, Avg.Score: -102.79, Time: 00:01:18\n",
      "Ep.: 5, Total Steps: 3475, Ep.Steps: 1600, Score: -88.98, Avg.Score: -100.49, Time: 00:02:30\n",
      "Ep.: 6, Total Steps: 3518, Ep.Steps: 43, Score: -107.26, Avg.Score: -101.46, Time: 00:02:32\n",
      "Ep.: 7, Total Steps: 3581, Ep.Steps: 63, Score: -107.01, Avg.Score: -102.15, Time: 00:02:35\n",
      "Ep.: 8, Total Steps: 3650, Ep.Steps: 69, Score: -99.62, Avg.Score: -101.87, Time: 00:02:38\n",
      "Ep.: 9, Total Steps: 5250, Ep.Steps: 1600, Score: -77.31, Avg.Score: -99.41, Time: 00:03:51\n",
      "Ep.: 10, Total Steps: 5333, Ep.Steps: 83, Score: -100.14, Avg.Score: -99.48, Time: 00:03:55\n",
      "Ep.: 11, Total Steps: 5414, Ep.Steps: 81, Score: -99.31, Avg.Score: -99.47, Time: 00:03:58\n",
      "Ep.: 12, Total Steps: 5509, Ep.Steps: 95, Score: -100.44, Avg.Score: -99.54, Time: 00:04:03\n",
      "Ep.: 13, Total Steps: 5604, Ep.Steps: 95, Score: -103.98, Avg.Score: -99.86, Time: 00:04:07\n",
      "Ep.: 14, Total Steps: 5681, Ep.Steps: 77, Score: -115.97, Avg.Score: -100.93, Time: 00:04:11\n",
      "Ep.: 15, Total Steps: 5761, Ep.Steps: 80, Score: -98.71, Avg.Score: -100.79, Time: 00:04:14\n",
      "Ep.: 16, Total Steps: 5962, Ep.Steps: 201, Score: -108.66, Avg.Score: -101.26, Time: 00:04:23\n",
      "Ep.: 17, Total Steps: 7562, Ep.Steps: 1600, Score: -81.63, Avg.Score: -100.17, Time: 00:05:36\n",
      "Ep.: 18, Total Steps: 7643, Ep.Steps: 81, Score: -121.82, Avg.Score: -101.31, Time: 00:05:40\n",
      "Ep.: 19, Total Steps: 9243, Ep.Steps: 1600, Score: -86.23, Avg.Score: -100.55, Time: 00:06:53\n",
      "Ep.: 20, Total Steps: 9330, Ep.Steps: 87, Score: -106.15, Avg.Score: -100.82, Time: 00:06:57\n",
      "Ep.: 21, Total Steps: 9408, Ep.Steps: 78, Score: -101.79, Avg.Score: -100.86, Time: 00:07:00\n",
      "Ep.: 22, Total Steps: 9477, Ep.Steps: 69, Score: -113.56, Avg.Score: -101.41, Time: 00:07:04\n",
      "Ep.: 23, Total Steps: 9556, Ep.Steps: 79, Score: -120.81, Avg.Score: -102.22, Time: 00:07:07\n",
      "Ep.: 24, Total Steps: 9626, Ep.Steps: 70, Score: -114.46, Avg.Score: -102.71, Time: 00:07:10\n",
      "Ep.: 25, Total Steps: 9689, Ep.Steps: 63, Score: -102.10, Avg.Score: -102.69, Time: 00:07:13\n",
      "Ep.: 26, Total Steps: 11289, Ep.Steps: 1600, Score: -121.95, Avg.Score: -103.40, Time: 00:08:30\n",
      "Ep.: 27, Total Steps: 11348, Ep.Steps: 59, Score: -100.05, Avg.Score: -103.28, Time: 00:08:33\n",
      "Ep.: 28, Total Steps: 11419, Ep.Steps: 71, Score: -99.07, Avg.Score: -103.14, Time: 00:08:36\n",
      "Ep.: 29, Total Steps: 11522, Ep.Steps: 103, Score: -103.27, Avg.Score: -103.14, Time: 00:08:41\n",
      "Ep.: 30, Total Steps: 11615, Ep.Steps: 93, Score: -101.17, Avg.Score: -103.08, Time: 00:08:46\n",
      "Ep.: 31, Total Steps: 11678, Ep.Steps: 63, Score: -97.82, Avg.Score: -102.91, Time: 00:08:49\n",
      "Ep.: 32, Total Steps: 11740, Ep.Steps: 62, Score: -106.68, Avg.Score: -103.03, Time: 00:08:52\n",
      "Ep.: 33, Total Steps: 11797, Ep.Steps: 57, Score: -105.98, Avg.Score: -103.11, Time: 00:08:55\n",
      "Ep.: 34, Total Steps: 11853, Ep.Steps: 56, Score: -105.23, Avg.Score: -103.17, Time: 00:08:57\n",
      "Ep.: 35, Total Steps: 11924, Ep.Steps: 71, Score: -107.12, Avg.Score: -103.28, Time: 00:09:01\n",
      "Ep.: 36, Total Steps: 12046, Ep.Steps: 122, Score: -105.54, Avg.Score: -103.34, Time: 00:09:07\n",
      "Ep.: 37, Total Steps: 12169, Ep.Steps: 123, Score: -104.43, Avg.Score: -103.37, Time: 00:09:12\n",
      "Ep.: 38, Total Steps: 12262, Ep.Steps: 93, Score: -107.52, Avg.Score: -103.48, Time: 00:09:17\n",
      "Ep.: 39, Total Steps: 12537, Ep.Steps: 275, Score: -117.13, Avg.Score: -103.82, Time: 00:09:30\n",
      "Ep.: 40, Total Steps: 12607, Ep.Steps: 70, Score: -108.33, Avg.Score: -103.93, Time: 00:09:34\n",
      "Ep.: 41, Total Steps: 12676, Ep.Steps: 69, Score: -111.87, Avg.Score: -104.12, Time: 00:09:37\n",
      "Ep.: 42, Total Steps: 12782, Ep.Steps: 106, Score: -105.33, Avg.Score: -104.15, Time: 00:09:43\n",
      "Ep.: 43, Total Steps: 13011, Ep.Steps: 229, Score: -97.20, Avg.Score: -103.99, Time: 00:09:54\n",
      "Ep.: 44, Total Steps: 13129, Ep.Steps: 118, Score: -101.09, Avg.Score: -103.93, Time: 00:09:59\n",
      "Ep.: 45, Total Steps: 13182, Ep.Steps: 53, Score: -102.58, Avg.Score: -103.90, Time: 00:10:02\n",
      "Ep.: 46, Total Steps: 13274, Ep.Steps: 92, Score: -102.83, Avg.Score: -103.87, Time: 00:10:06\n",
      "Ep.: 47, Total Steps: 13359, Ep.Steps: 85, Score: -105.30, Avg.Score: -103.90, Time: 00:10:11\n",
      "Ep.: 48, Total Steps: 13447, Ep.Steps: 88, Score: -99.66, Avg.Score: -103.82, Time: 00:10:15\n",
      "Ep.: 49, Total Steps: 13555, Ep.Steps: 108, Score: -106.72, Avg.Score: -103.88, Time: 00:10:20\n",
      "Ep.: 50, Total Steps: 13662, Ep.Steps: 107, Score: -103.27, Avg.Score: -103.86, Time: 00:10:25\n",
      "Ep.: 51, Total Steps: 13772, Ep.Steps: 110, Score: -109.24, Avg.Score: -103.97, Time: 00:10:30\n",
      "Ep.: 52, Total Steps: 13840, Ep.Steps: 68, Score: -97.51, Avg.Score: -103.84, Time: 00:10:34\n",
      "Ep.: 53, Total Steps: 13904, Ep.Steps: 64, Score: -97.65, Avg.Score: -103.73, Time: 00:10:37\n",
      "Ep.: 54, Total Steps: 13946, Ep.Steps: 42, Score: -104.48, Avg.Score: -103.74, Time: 00:10:39\n",
      "Ep.: 55, Total Steps: 13990, Ep.Steps: 44, Score: -101.67, Avg.Score: -103.71, Time: 00:10:41\n",
      "Ep.: 56, Total Steps: 14033, Ep.Steps: 43, Score: -102.42, Avg.Score: -103.68, Time: 00:10:43\n",
      "Ep.: 57, Total Steps: 14087, Ep.Steps: 54, Score: -103.12, Avg.Score: -103.67, Time: 00:10:46\n",
      "Ep.: 58, Total Steps: 14132, Ep.Steps: 45, Score: -103.53, Avg.Score: -103.67, Time: 00:10:48\n",
      "Ep.: 59, Total Steps: 14181, Ep.Steps: 49, Score: -102.77, Avg.Score: -103.66, Time: 00:10:51\n",
      "Ep.: 60, Total Steps: 14242, Ep.Steps: 61, Score: -97.81, Avg.Score: -103.56, Time: 00:10:54\n",
      "Ep.: 61, Total Steps: 14308, Ep.Steps: 66, Score: -100.93, Avg.Score: -103.52, Time: 00:10:57\n",
      "Ep.: 62, Total Steps: 14374, Ep.Steps: 66, Score: -104.90, Avg.Score: -103.54, Time: 00:11:00\n",
      "Ep.: 63, Total Steps: 14434, Ep.Steps: 60, Score: -102.13, Avg.Score: -103.52, Time: 00:11:03\n",
      "Ep.: 64, Total Steps: 14496, Ep.Steps: 62, Score: -98.73, Avg.Score: -103.44, Time: 00:11:06\n",
      "Ep.: 65, Total Steps: 14554, Ep.Steps: 58, Score: -101.96, Avg.Score: -103.42, Time: 00:11:09\n",
      "Ep.: 66, Total Steps: 14614, Ep.Steps: 60, Score: -101.74, Avg.Score: -103.40, Time: 00:11:12\n",
      "Ep.: 67, Total Steps: 14682, Ep.Steps: 68, Score: -104.83, Avg.Score: -103.42, Time: 00:11:15\n",
      "Ep.: 68, Total Steps: 14747, Ep.Steps: 65, Score: -102.72, Avg.Score: -103.41, Time: 00:11:18\n",
      "Ep.: 69, Total Steps: 14812, Ep.Steps: 65, Score: -102.50, Avg.Score: -103.40, Time: 00:11:22\n",
      "Ep.: 70, Total Steps: 14885, Ep.Steps: 73, Score: -104.33, Avg.Score: -103.41, Time: 00:11:25\n",
      "Ep.: 71, Total Steps: 14938, Ep.Steps: 53, Score: -108.27, Avg.Score: -103.48, Time: 00:11:28\n",
      "Ep.: 72, Total Steps: 15003, Ep.Steps: 65, Score: -105.71, Avg.Score: -103.51, Time: 00:11:31\n",
      "Ep.: 73, Total Steps: 15060, Ep.Steps: 57, Score: -102.44, Avg.Score: -103.49, Time: 00:11:34\n",
      "Ep.: 74, Total Steps: 15184, Ep.Steps: 124, Score: -109.36, Avg.Score: -103.57, Time: 00:11:40\n",
      "Ep.: 75, Total Steps: 15243, Ep.Steps: 59, Score: -106.06, Avg.Score: -103.60, Time: 00:11:43\n",
      "Ep.: 76, Total Steps: 15301, Ep.Steps: 58, Score: -102.05, Avg.Score: -103.58, Time: 00:11:46\n",
      "Ep.: 77, Total Steps: 15369, Ep.Steps: 68, Score: -104.84, Avg.Score: -103.60, Time: 00:11:49\n",
      "Ep.: 78, Total Steps: 15420, Ep.Steps: 51, Score: -102.89, Avg.Score: -103.59, Time: 00:11:51\n",
      "Ep.: 79, Total Steps: 15494, Ep.Steps: 74, Score: -102.77, Avg.Score: -103.58, Time: 00:11:55\n",
      "Ep.: 80, Total Steps: 15546, Ep.Steps: 52, Score: -104.81, Avg.Score: -103.60, Time: 00:11:58\n",
      "Ep.: 81, Total Steps: 15612, Ep.Steps: 66, Score: -104.16, Avg.Score: -103.60, Time: 00:12:02\n",
      "Ep.: 82, Total Steps: 15666, Ep.Steps: 54, Score: -104.04, Avg.Score: -103.61, Time: 00:12:04\n",
      "Ep.: 83, Total Steps: 15734, Ep.Steps: 68, Score: -105.19, Avg.Score: -103.63, Time: 00:12:08\n",
      "Ep.: 84, Total Steps: 15787, Ep.Steps: 53, Score: -103.78, Avg.Score: -103.63, Time: 00:12:11\n",
      "Ep.: 85, Total Steps: 15849, Ep.Steps: 62, Score: -102.60, Avg.Score: -103.62, Time: 00:12:14\n",
      "Ep.: 86, Total Steps: 15907, Ep.Steps: 58, Score: -103.26, Avg.Score: -103.61, Time: 00:12:17\n",
      "Ep.: 87, Total Steps: 15969, Ep.Steps: 62, Score: -101.13, Avg.Score: -103.58, Time: 00:12:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 88, Total Steps: 16028, Ep.Steps: 59, Score: -103.86, Avg.Score: -103.59, Time: 00:12:23\n",
      "Ep.: 89, Total Steps: 16086, Ep.Steps: 58, Score: -103.41, Avg.Score: -103.59, Time: 00:12:26\n",
      "Ep.: 90, Total Steps: 16161, Ep.Steps: 75, Score: -103.59, Avg.Score: -103.59, Time: 00:12:30\n",
      "Ep.: 91, Total Steps: 16228, Ep.Steps: 67, Score: -103.46, Avg.Score: -103.58, Time: 00:12:34\n",
      "Ep.: 92, Total Steps: 16315, Ep.Steps: 87, Score: -102.36, Avg.Score: -103.57, Time: 00:12:38\n",
      "Ep.: 93, Total Steps: 16378, Ep.Steps: 63, Score: -102.61, Avg.Score: -103.56, Time: 00:12:42\n",
      "Ep.: 94, Total Steps: 16465, Ep.Steps: 87, Score: -99.48, Avg.Score: -103.52, Time: 00:12:46\n",
      "Ep.: 95, Total Steps: 16522, Ep.Steps: 57, Score: -103.48, Avg.Score: -103.52, Time: 00:12:49\n",
      "Ep.: 96, Total Steps: 16583, Ep.Steps: 61, Score: -99.72, Avg.Score: -103.48, Time: 00:12:52\n",
      "Ep.: 97, Total Steps: 16663, Ep.Steps: 80, Score: -98.91, Avg.Score: -103.43, Time: 00:12:56\n",
      "Ep.: 98, Total Steps: 16723, Ep.Steps: 60, Score: -102.68, Avg.Score: -103.42, Time: 00:12:59\n",
      "Ep.: 99, Total Steps: 16781, Ep.Steps: 58, Score: -106.03, Avg.Score: -103.45, Time: 00:13:02\n",
      "Ep.: 100, Total Steps: 16830, Ep.Steps: 49, Score: -106.13, Avg.Score: -103.53, Time: 00:13:04\n",
      "Ep.: 101, Total Steps: 16889, Ep.Steps: 59, Score: -104.70, Avg.Score: -103.43, Time: 00:13:07\n",
      "Ep.: 102, Total Steps: 16951, Ep.Steps: 62, Score: -103.91, Avg.Score: -103.46, Time: 00:13:10\n",
      "Ep.: 103, Total Steps: 17015, Ep.Steps: 64, Score: -111.22, Avg.Score: -103.70, Time: 00:13:13\n",
      "Ep.: 104, Total Steps: 17086, Ep.Steps: 71, Score: -105.75, Avg.Score: -103.63, Time: 00:13:16\n",
      "Ep.: 105, Total Steps: 17145, Ep.Steps: 59, Score: -103.11, Avg.Score: -103.77, Time: 00:13:19\n",
      "Ep.: 106, Total Steps: 17195, Ep.Steps: 50, Score: -104.40, Avg.Score: -103.74, Time: 00:13:22\n",
      "Ep.: 107, Total Steps: 17244, Ep.Steps: 49, Score: -103.77, Avg.Score: -103.71, Time: 00:13:24\n",
      "Ep.: 108, Total Steps: 17298, Ep.Steps: 54, Score: -104.24, Avg.Score: -103.75, Time: 00:13:27\n",
      "Ep.: 109, Total Steps: 17352, Ep.Steps: 54, Score: -102.96, Avg.Score: -104.01, Time: 00:13:30\n",
      "Ep.: 110, Total Steps: 17462, Ep.Steps: 110, Score: -102.68, Avg.Score: -104.04, Time: 00:13:38\n",
      "Ep.: 111, Total Steps: 17515, Ep.Steps: 53, Score: -105.00, Avg.Score: -104.09, Time: 00:13:41\n",
      "Ep.: 112, Total Steps: 17569, Ep.Steps: 54, Score: -103.32, Avg.Score: -104.12, Time: 00:13:44\n",
      "Ep.: 113, Total Steps: 17617, Ep.Steps: 48, Score: -101.44, Avg.Score: -104.10, Time: 00:13:47\n",
      "Ep.: 114, Total Steps: 17669, Ep.Steps: 52, Score: -103.60, Avg.Score: -103.97, Time: 00:13:50\n",
      "Ep.: 115, Total Steps: 17721, Ep.Steps: 52, Score: -101.53, Avg.Score: -104.00, Time: 00:13:53\n",
      "Ep.: 116, Total Steps: 17781, Ep.Steps: 60, Score: -104.88, Avg.Score: -103.96, Time: 00:13:56\n",
      "Ep.: 117, Total Steps: 17864, Ep.Steps: 83, Score: -115.17, Avg.Score: -104.30, Time: 00:14:01\n",
      "Ep.: 118, Total Steps: 17914, Ep.Steps: 50, Score: -102.86, Avg.Score: -104.11, Time: 00:14:03\n",
      "Ep.: 119, Total Steps: 17979, Ep.Steps: 65, Score: -102.68, Avg.Score: -104.27, Time: 00:14:07\n",
      "Ep.: 120, Total Steps: 18040, Ep.Steps: 61, Score: -102.81, Avg.Score: -104.24, Time: 00:14:10\n",
      "Ep.: 121, Total Steps: 18103, Ep.Steps: 63, Score: -102.84, Avg.Score: -104.25, Time: 00:14:13\n",
      "Ep.: 122, Total Steps: 18175, Ep.Steps: 72, Score: -99.58, Avg.Score: -104.11, Time: 00:14:17\n",
      "Ep.: 123, Total Steps: 18235, Ep.Steps: 60, Score: -106.74, Avg.Score: -103.97, Time: 00:14:20\n",
      "Ep.: 124, Total Steps: 18295, Ep.Steps: 60, Score: -104.54, Avg.Score: -103.87, Time: 00:14:24\n",
      "Ep.: 125, Total Steps: 18353, Ep.Steps: 58, Score: -109.46, Avg.Score: -103.94, Time: 00:14:27\n",
      "Ep.: 126, Total Steps: 18421, Ep.Steps: 68, Score: -106.03, Avg.Score: -103.78, Time: 00:14:30\n",
      "Ep.: 127, Total Steps: 18619, Ep.Steps: 198, Score: -108.88, Avg.Score: -103.87, Time: 00:14:46\n",
      "Ep.: 128, Total Steps: 18704, Ep.Steps: 85, Score: -104.42, Avg.Score: -103.93, Time: 00:14:51\n",
      "Ep.: 129, Total Steps: 18762, Ep.Steps: 58, Score: -110.27, Avg.Score: -104.00, Time: 00:14:55\n",
      "Ep.: 130, Total Steps: 18828, Ep.Steps: 66, Score: -104.81, Avg.Score: -104.03, Time: 00:14:58\n",
      "Ep.: 131, Total Steps: 19050, Ep.Steps: 222, Score: -112.48, Avg.Score: -104.18, Time: 00:15:11\n",
      "Ep.: 132, Total Steps: 19119, Ep.Steps: 69, Score: -101.66, Avg.Score: -104.13, Time: 00:15:15\n",
      "Ep.: 133, Total Steps: 19176, Ep.Steps: 57, Score: -102.35, Avg.Score: -104.09, Time: 00:15:18\n",
      "Ep.: 134, Total Steps: 19298, Ep.Steps: 122, Score: -101.82, Avg.Score: -104.06, Time: 00:15:25\n",
      "Ep.: 135, Total Steps: 19387, Ep.Steps: 89, Score: -99.36, Avg.Score: -103.98, Time: 00:15:29\n",
      "Ep.: 136, Total Steps: 19448, Ep.Steps: 61, Score: -108.51, Avg.Score: -104.01, Time: 00:15:33\n",
      "Ep.: 137, Total Steps: 19634, Ep.Steps: 186, Score: -106.26, Avg.Score: -104.03, Time: 00:15:43\n",
      "Ep.: 138, Total Steps: 19721, Ep.Steps: 87, Score: -98.61, Avg.Score: -103.94, Time: 00:15:48\n",
      "Ep.: 139, Total Steps: 19807, Ep.Steps: 86, Score: -102.22, Avg.Score: -103.79, Time: 00:15:53\n",
      "Ep.: 140, Total Steps: 19965, Ep.Steps: 158, Score: -105.00, Avg.Score: -103.76, Time: 00:16:02\n",
      "Ep.: 141, Total Steps: 20178, Ep.Steps: 213, Score: -106.71, Avg.Score: -103.71, Time: 00:16:13\n",
      "Ep.: 142, Total Steps: 20245, Ep.Steps: 67, Score: -106.56, Avg.Score: -103.72, Time: 00:16:17\n",
      "Ep.: 143, Total Steps: 20303, Ep.Steps: 58, Score: -104.55, Avg.Score: -103.79, Time: 00:16:20\n",
      "Ep.: 144, Total Steps: 20389, Ep.Steps: 86, Score: -99.48, Avg.Score: -103.78, Time: 00:16:25\n",
      "Ep.: 145, Total Steps: 20463, Ep.Steps: 74, Score: -99.42, Avg.Score: -103.74, Time: 00:16:29\n",
      "Ep.: 146, Total Steps: 20563, Ep.Steps: 100, Score: -100.65, Avg.Score: -103.72, Time: 00:16:35\n",
      "Ep.: 147, Total Steps: 22163, Ep.Steps: 1600, Score: -80.08, Avg.Score: -103.47, Time: 00:18:02\n",
      "Ep.: 148, Total Steps: 22242, Ep.Steps: 79, Score: -99.55, Avg.Score: -103.47, Time: 00:18:07\n",
      "Ep.: 149, Total Steps: 22330, Ep.Steps: 88, Score: -107.05, Avg.Score: -103.47, Time: 00:18:11\n",
      "Ep.: 150, Total Steps: 22445, Ep.Steps: 115, Score: -135.64, Avg.Score: -103.80, Time: 00:18:18\n",
      "Ep.: 151, Total Steps: 22533, Ep.Steps: 88, Score: -99.95, Avg.Score: -103.70, Time: 00:18:23\n",
      "Ep.: 152, Total Steps: 22586, Ep.Steps: 53, Score: -108.48, Avg.Score: -103.81, Time: 00:18:25\n",
      "Ep.: 153, Total Steps: 22634, Ep.Steps: 48, Score: -108.22, Avg.Score: -103.92, Time: 00:18:28\n",
      "Ep.: 154, Total Steps: 22723, Ep.Steps: 89, Score: -98.19, Avg.Score: -103.86, Time: 00:18:33\n",
      "Ep.: 155, Total Steps: 22779, Ep.Steps: 56, Score: -107.65, Avg.Score: -103.92, Time: 00:18:36\n",
      "Ep.: 156, Total Steps: 22890, Ep.Steps: 111, Score: -100.17, Avg.Score: -103.89, Time: 00:18:42\n",
      "Ep.: 157, Total Steps: 22962, Ep.Steps: 72, Score: -99.55, Avg.Score: -103.86, Time: 00:18:49\n",
      "Ep.: 158, Total Steps: 23031, Ep.Steps: 69, Score: -123.73, Avg.Score: -104.06, Time: 00:18:53\n",
      "Ep.: 159, Total Steps: 23085, Ep.Steps: 54, Score: -119.82, Avg.Score: -104.23, Time: 00:18:56\n",
      "Ep.: 160, Total Steps: 23174, Ep.Steps: 89, Score: -100.63, Avg.Score: -104.26, Time: 00:19:01\n",
      "Ep.: 161, Total Steps: 23246, Ep.Steps: 72, Score: -98.95, Avg.Score: -104.24, Time: 00:19:05\n",
      "Ep.: 162, Total Steps: 23383, Ep.Steps: 137, Score: -102.08, Avg.Score: -104.21, Time: 00:19:12\n",
      "Ep.: 163, Total Steps: 23463, Ep.Steps: 80, Score: -98.80, Avg.Score: -104.18, Time: 00:19:16\n",
      "Ep.: 164, Total Steps: 23569, Ep.Steps: 106, Score: -98.50, Avg.Score: -104.17, Time: 00:19:22\n",
      "Ep.: 165, Total Steps: 23699, Ep.Steps: 130, Score: -100.60, Avg.Score: -104.16, Time: 00:19:29\n",
      "Ep.: 166, Total Steps: 23891, Ep.Steps: 192, Score: -121.16, Avg.Score: -104.36, Time: 00:19:40\n",
      "Ep.: 167, Total Steps: 24043, Ep.Steps: 152, Score: -105.16, Avg.Score: -104.36, Time: 00:19:48\n",
      "Ep.: 168, Total Steps: 24149, Ep.Steps: 106, Score: -102.14, Avg.Score: -104.35, Time: 00:19:54\n",
      "Ep.: 169, Total Steps: 24248, Ep.Steps: 99, Score: -103.77, Avg.Score: -104.37, Time: 00:19:59\n",
      "Ep.: 170, Total Steps: 24365, Ep.Steps: 117, Score: -105.82, Avg.Score: -104.38, Time: 00:20:06\n",
      "Ep.: 171, Total Steps: 24502, Ep.Steps: 137, Score: -100.55, Avg.Score: -104.30, Time: 00:20:13\n",
      "Ep.: 172, Total Steps: 24599, Ep.Steps: 97, Score: -106.95, Avg.Score: -104.32, Time: 00:20:18\n",
      "Ep.: 173, Total Steps: 24693, Ep.Steps: 94, Score: -109.83, Avg.Score: -104.39, Time: 00:20:22\n",
      "Ep.: 174, Total Steps: 26293, Ep.Steps: 1600, Score: -97.43, Avg.Score: -104.27, Time: 00:21:45\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 175, Total Steps: 26385, Ep.Steps: 92, Score: -125.06, Avg.Score: -104.46, Time: 00:21:49\n",
      "Ep.: 176, Total Steps: 26427, Ep.Steps: 42, Score: -114.65, Avg.Score: -104.59, Time: 00:21:51\n",
      "Ep.: 177, Total Steps: 28027, Ep.Steps: 1600, Score: -122.14, Avg.Score: -104.76, Time: 00:23:16\n",
      "Ep.: 178, Total Steps: 29627, Ep.Steps: 1600, Score: -108.91, Avg.Score: -104.82, Time: 00:24:36\n",
      "Ep.: 179, Total Steps: 29709, Ep.Steps: 82, Score: -115.83, Avg.Score: -104.95, Time: 00:24:40\n",
      "Ep.: 180, Total Steps: 31309, Ep.Steps: 1600, Score: -112.30, Avg.Score: -105.02, Time: 00:26:02\n",
      "Ep.: 181, Total Steps: 32909, Ep.Steps: 1600, Score: -101.63, Avg.Score: -105.00, Time: 00:27:23\n",
      "Ep.: 182, Total Steps: 34509, Ep.Steps: 1600, Score: -101.16, Avg.Score: -104.97, Time: 00:28:46\n",
      "Ep.: 183, Total Steps: 34611, Ep.Steps: 102, Score: -113.91, Avg.Score: -105.06, Time: 00:28:51\n",
      "Ep.: 184, Total Steps: 34691, Ep.Steps: 80, Score: -117.59, Avg.Score: -105.20, Time: 00:28:55\n",
      "Ep.: 185, Total Steps: 36291, Ep.Steps: 1600, Score: -81.48, Avg.Score: -104.98, Time: 00:30:19\n",
      "Ep.: 186, Total Steps: 37891, Ep.Steps: 1600, Score: -70.19, Avg.Score: -104.65, Time: 00:31:41\n",
      "Ep.: 187, Total Steps: 38245, Ep.Steps: 354, Score: -124.42, Avg.Score: -104.89, Time: 00:31:59\n",
      "Ep.: 188, Total Steps: 38538, Ep.Steps: 293, Score: -122.75, Avg.Score: -105.08, Time: 00:32:14\n",
      "Ep.: 189, Total Steps: 40138, Ep.Steps: 1600, Score: -84.17, Avg.Score: -104.88, Time: 00:33:37\n",
      "Ep.: 190, Total Steps: 41738, Ep.Steps: 1600, Score: -52.00, Avg.Score: -104.37, Time: 00:34:59\n",
      "Ep.: 191, Total Steps: 41817, Ep.Steps: 79, Score: -111.45, Avg.Score: -104.45, Time: 00:35:03\n",
      "Ep.: 192, Total Steps: 43417, Ep.Steps: 1600, Score: -68.52, Avg.Score: -104.11, Time: 00:36:25\n",
      "Ep.: 193, Total Steps: 45017, Ep.Steps: 1600, Score: -75.55, Avg.Score: -103.84, Time: 00:37:47\n",
      "Ep.: 194, Total Steps: 45091, Ep.Steps: 74, Score: -105.94, Avg.Score: -103.90, Time: 00:37:51\n",
      "Ep.: 195, Total Steps: 46691, Ep.Steps: 1600, Score: -92.01, Avg.Score: -103.79, Time: 00:39:15\n",
      "Ep.: 196, Total Steps: 47211, Ep.Steps: 520, Score: -115.55, Avg.Score: -103.95, Time: 00:39:42\n",
      "Ep.: 197, Total Steps: 47283, Ep.Steps: 72, Score: -110.20, Avg.Score: -104.06, Time: 00:39:46\n",
      "Ep.: 198, Total Steps: 47358, Ep.Steps: 75, Score: -109.28, Avg.Score: -104.13, Time: 00:39:52\n",
      "Ep.: 199, Total Steps: 47426, Ep.Steps: 68, Score: -106.23, Avg.Score: -104.13, Time: 00:39:55\n",
      "Ep.: 200, Total Steps: 47484, Ep.Steps: 58, Score: -107.58, Avg.Score: -104.14, Time: 00:39:58\n",
      "Ep.: 201, Total Steps: 47556, Ep.Steps: 72, Score: -111.76, Avg.Score: -104.21, Time: 00:40:02\n",
      "Ep.: 202, Total Steps: 47627, Ep.Steps: 71, Score: -104.70, Avg.Score: -104.22, Time: 00:40:07\n",
      "Ep.: 203, Total Steps: 47917, Ep.Steps: 290, Score: -114.39, Avg.Score: -104.25, Time: 00:40:22\n",
      "Ep.: 204, Total Steps: 47969, Ep.Steps: 52, Score: -106.45, Avg.Score: -104.26, Time: 00:40:25\n",
      "Ep.: 205, Total Steps: 48046, Ep.Steps: 77, Score: -111.39, Avg.Score: -104.34, Time: 00:40:28\n",
      "Ep.: 206, Total Steps: 48120, Ep.Steps: 74, Score: -105.64, Avg.Score: -104.35, Time: 00:40:32\n",
      "Ep.: 207, Total Steps: 48198, Ep.Steps: 78, Score: -104.27, Avg.Score: -104.36, Time: 00:40:36\n",
      "Ep.: 208, Total Steps: 48264, Ep.Steps: 66, Score: -103.63, Avg.Score: -104.35, Time: 00:40:40\n",
      "Ep.: 209, Total Steps: 48333, Ep.Steps: 69, Score: -104.69, Avg.Score: -104.37, Time: 00:40:43\n",
      "Ep.: 210, Total Steps: 48454, Ep.Steps: 121, Score: -101.81, Avg.Score: -104.36, Time: 00:40:50\n",
      "Ep.: 211, Total Steps: 48564, Ep.Steps: 110, Score: -98.06, Avg.Score: -104.29, Time: 00:40:56\n",
      "Ep.: 212, Total Steps: 48657, Ep.Steps: 93, Score: -102.80, Avg.Score: -104.29, Time: 00:41:00\n",
      "Ep.: 213, Total Steps: 48763, Ep.Steps: 106, Score: -104.54, Avg.Score: -104.32, Time: 00:41:06\n",
      "Ep.: 214, Total Steps: 48843, Ep.Steps: 80, Score: -102.83, Avg.Score: -104.31, Time: 00:41:10\n",
      "Ep.: 215, Total Steps: 48947, Ep.Steps: 104, Score: -98.06, Avg.Score: -104.28, Time: 00:41:15\n",
      "Ep.: 216, Total Steps: 49026, Ep.Steps: 79, Score: -102.39, Avg.Score: -104.25, Time: 00:41:19\n",
      "Ep.: 217, Total Steps: 49098, Ep.Steps: 72, Score: -101.28, Avg.Score: -104.11, Time: 00:41:23\n",
      "Ep.: 218, Total Steps: 49207, Ep.Steps: 109, Score: -95.10, Avg.Score: -104.03, Time: 00:41:29\n",
      "Ep.: 219, Total Steps: 49384, Ep.Steps: 177, Score: -98.08, Avg.Score: -103.99, Time: 00:41:38\n",
      "Ep.: 220, Total Steps: 49614, Ep.Steps: 230, Score: -88.57, Avg.Score: -103.85, Time: 00:41:51\n",
      "Ep.: 221, Total Steps: 49806, Ep.Steps: 192, Score: -98.15, Avg.Score: -103.80, Time: 00:42:03\n",
      "Ep.: 222, Total Steps: 49902, Ep.Steps: 96, Score: -100.44, Avg.Score: -103.81, Time: 00:42:08\n",
      "Ep.: 223, Total Steps: 50037, Ep.Steps: 135, Score: -102.64, Avg.Score: -103.77, Time: 00:42:15\n",
      "Ep.: 224, Total Steps: 50222, Ep.Steps: 185, Score: -97.75, Avg.Score: -103.70, Time: 00:42:25\n",
      "Ep.: 225, Total Steps: 50391, Ep.Steps: 169, Score: -94.11, Avg.Score: -103.55, Time: 00:42:34\n",
      "Ep.: 226, Total Steps: 50632, Ep.Steps: 241, Score: -100.80, Avg.Score: -103.49, Time: 00:42:46\n",
      "Ep.: 227, Total Steps: 50842, Ep.Steps: 210, Score: -93.69, Avg.Score: -103.34, Time: 00:42:57\n",
      "Ep.: 228, Total Steps: 51118, Ep.Steps: 276, Score: -92.78, Avg.Score: -103.23, Time: 00:43:11\n",
      "Ep.: 229, Total Steps: 51191, Ep.Steps: 73, Score: -107.26, Avg.Score: -103.19, Time: 00:43:14\n",
      "Ep.: 230, Total Steps: 51281, Ep.Steps: 90, Score: -104.74, Avg.Score: -103.19, Time: 00:43:19\n",
      "Ep.: 231, Total Steps: 51338, Ep.Steps: 57, Score: -114.43, Avg.Score: -103.21, Time: 00:43:22\n",
      "Ep.: 232, Total Steps: 51400, Ep.Steps: 62, Score: -109.41, Avg.Score: -103.29, Time: 00:43:25\n",
      "Ep.: 233, Total Steps: 51495, Ep.Steps: 95, Score: -97.79, Avg.Score: -103.25, Time: 00:43:31\n",
      "Ep.: 234, Total Steps: 51617, Ep.Steps: 122, Score: -106.12, Avg.Score: -103.29, Time: 00:43:37\n",
      "Ep.: 235, Total Steps: 51689, Ep.Steps: 72, Score: -106.44, Avg.Score: -103.36, Time: 00:43:41\n",
      "Ep.: 236, Total Steps: 51763, Ep.Steps: 74, Score: -104.18, Avg.Score: -103.32, Time: 00:43:44\n",
      "Ep.: 237, Total Steps: 51837, Ep.Steps: 74, Score: -106.06, Avg.Score: -103.31, Time: 00:43:48\n",
      "Ep.: 238, Total Steps: 51936, Ep.Steps: 99, Score: -99.03, Avg.Score: -103.32, Time: 00:43:54\n",
      "Ep.: 239, Total Steps: 52025, Ep.Steps: 89, Score: -98.75, Avg.Score: -103.28, Time: 00:43:58\n",
      "Ep.: 240, Total Steps: 52135, Ep.Steps: 110, Score: -97.50, Avg.Score: -103.21, Time: 00:44:06\n",
      "Ep.: 241, Total Steps: 52206, Ep.Steps: 71, Score: -104.64, Avg.Score: -103.19, Time: 00:44:10\n",
      "Ep.: 242, Total Steps: 52278, Ep.Steps: 72, Score: -104.10, Avg.Score: -103.16, Time: 00:44:14\n",
      "Ep.: 243, Total Steps: 52474, Ep.Steps: 196, Score: -106.14, Avg.Score: -103.18, Time: 00:44:25\n",
      "Ep.: 244, Total Steps: 52552, Ep.Steps: 78, Score: -100.54, Avg.Score: -103.19, Time: 00:44:30\n",
      "Ep.: 245, Total Steps: 52702, Ep.Steps: 150, Score: -100.82, Avg.Score: -103.20, Time: 00:44:39\n",
      "Ep.: 246, Total Steps: 53261, Ep.Steps: 559, Score: -103.80, Avg.Score: -103.24, Time: 00:45:13\n",
      "Ep.: 247, Total Steps: 53373, Ep.Steps: 112, Score: -117.91, Avg.Score: -103.61, Time: 00:45:19\n",
      "Ep.: 248, Total Steps: 53446, Ep.Steps: 73, Score: -99.90, Avg.Score: -103.62, Time: 00:45:23\n",
      "Ep.: 249, Total Steps: 53566, Ep.Steps: 120, Score: -94.26, Avg.Score: -103.49, Time: 00:45:29\n",
      "Ep.: 250, Total Steps: 53700, Ep.Steps: 134, Score: -94.07, Avg.Score: -103.07, Time: 00:45:36\n",
      "Ep.: 251, Total Steps: 54160, Ep.Steps: 460, Score: -108.46, Avg.Score: -103.16, Time: 00:46:00\n",
      "Ep.: 252, Total Steps: 55760, Ep.Steps: 1600, Score: -67.94, Avg.Score: -102.75, Time: 00:47:23\n",
      "Ep.: 253, Total Steps: 57360, Ep.Steps: 1600, Score: -78.39, Avg.Score: -102.46, Time: 00:48:44\n",
      "Ep.: 254, Total Steps: 58960, Ep.Steps: 1600, Score: -84.43, Avg.Score: -102.32, Time: 00:50:05\n",
      "Ep.: 255, Total Steps: 60560, Ep.Steps: 1600, Score: -80.85, Avg.Score: -102.05, Time: 00:51:28\n",
      "Ep.: 256, Total Steps: 62160, Ep.Steps: 1600, Score: -65.71, Avg.Score: -101.70, Time: 00:52:49\n",
      "Ep.: 257, Total Steps: 63760, Ep.Steps: 1600, Score: -62.42, Avg.Score: -101.33, Time: 00:54:10\n",
      "Ep.: 258, Total Steps: 65360, Ep.Steps: 1600, Score: -42.06, Avg.Score: -100.52, Time: 00:55:32\n",
      "Ep.: 259, Total Steps: 66960, Ep.Steps: 1600, Score: -43.66, Avg.Score: -99.76, Time: 00:56:59\n",
      "Ep.: 260, Total Steps: 68560, Ep.Steps: 1600, Score: -41.11, Avg.Score: -99.16, Time: 00:58:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 261, Total Steps: 70160, Ep.Steps: 1600, Score: -16.02, Avg.Score: -98.33, Time: 00:59:51\n",
      "Ep.: 262, Total Steps: 71760, Ep.Steps: 1600, Score: -116.96, Avg.Score: -98.48, Time: 01:01:15\n",
      "Ep.: 263, Total Steps: 73360, Ep.Steps: 1600, Score: -79.51, Avg.Score: -98.29, Time: 01:02:37\n",
      "Ep.: 264, Total Steps: 74960, Ep.Steps: 1600, Score: -57.48, Avg.Score: -97.88, Time: 01:04:00\n",
      "Ep.: 265, Total Steps: 76560, Ep.Steps: 1600, Score: -44.33, Avg.Score: -97.31, Time: 01:05:23\n",
      "Ep.: 266, Total Steps: 78160, Ep.Steps: 1600, Score: -39.95, Avg.Score: -96.50, Time: 01:06:55\n",
      "Ep.: 267, Total Steps: 79424, Ep.Steps: 1264, Score: -84.74, Avg.Score: -96.30, Time: 01:08:04\n",
      "Ep.: 268, Total Steps: 81024, Ep.Steps: 1600, Score: 16.84, Avg.Score: -95.11, Time: 01:09:29\n",
      "Ep.: 269, Total Steps: 81710, Ep.Steps: 686, Score: -83.14, Avg.Score: -94.90, Time: 01:10:05\n",
      "Ep.: 270, Total Steps: 82004, Ep.Steps: 294, Score: -84.67, Avg.Score: -94.69, Time: 01:10:20\n",
      "Ep.: 271, Total Steps: 82361, Ep.Steps: 357, Score: -81.11, Avg.Score: -94.50, Time: 01:10:39\n",
      "Ep.: 272, Total Steps: 82533, Ep.Steps: 172, Score: -101.50, Avg.Score: -94.44, Time: 01:10:48\n",
      "Ep.: 273, Total Steps: 82677, Ep.Steps: 144, Score: -96.09, Avg.Score: -94.30, Time: 01:10:56\n",
      "Ep.: 274, Total Steps: 83560, Ep.Steps: 883, Score: -70.98, Avg.Score: -94.04, Time: 01:11:43\n",
      "Ep.: 275, Total Steps: 83856, Ep.Steps: 296, Score: -91.56, Avg.Score: -93.70, Time: 01:12:00\n",
      "Ep.: 276, Total Steps: 85456, Ep.Steps: 1600, Score: -17.72, Avg.Score: -92.73, Time: 01:13:26\n",
      "Ep.: 277, Total Steps: 85728, Ep.Steps: 272, Score: -96.28, Avg.Score: -92.48, Time: 01:13:41\n",
      "Ep.: 278, Total Steps: 87187, Ep.Steps: 1459, Score: -52.42, Avg.Score: -91.91, Time: 01:15:00\n",
      "Ep.: 279, Total Steps: 88787, Ep.Steps: 1600, Score: 14.38, Avg.Score: -90.61, Time: 01:16:27\n",
      "Ep.: 280, Total Steps: 90387, Ep.Steps: 1600, Score: 61.57, Avg.Score: -88.87, Time: 01:17:54\n",
      "Ep.: 281, Total Steps: 91987, Ep.Steps: 1600, Score: 109.96, Avg.Score: -86.75, Time: 01:19:16\n",
      "Ep.: 282, Total Steps: 93587, Ep.Steps: 1600, Score: 123.43, Avg.Score: -84.51, Time: 01:20:39\n",
      "Ep.: 283, Total Steps: 94404, Ep.Steps: 817, Score: -41.64, Avg.Score: -83.79, Time: 01:21:21\n",
      "Ep.: 284, Total Steps: 96004, Ep.Steps: 1600, Score: 156.73, Avg.Score: -81.04, Time: 01:22:44\n",
      "Ep.: 285, Total Steps: 96092, Ep.Steps: 88, Score: -111.21, Avg.Score: -81.34, Time: 01:22:48\n",
      "Ep.: 286, Total Steps: 96274, Ep.Steps: 182, Score: -123.33, Avg.Score: -81.87, Time: 01:22:58\n",
      "Ep.: 287, Total Steps: 96398, Ep.Steps: 124, Score: -106.39, Avg.Score: -81.69, Time: 01:23:04\n",
      "Ep.: 288, Total Steps: 96528, Ep.Steps: 130, Score: -124.01, Avg.Score: -81.70, Time: 01:23:11\n",
      "Ep.: 289, Total Steps: 98128, Ep.Steps: 1600, Score: 164.35, Avg.Score: -79.22, Time: 01:24:33\n",
      "Ep.: 290, Total Steps: 99728, Ep.Steps: 1600, Score: 176.42, Avg.Score: -76.93, Time: 01:25:56\n",
      "Ep.: 291, Total Steps: 100773, Ep.Steps: 1045, Score: 17.10, Avg.Score: -75.65, Time: 01:26:50\n",
      "Ep.: 292, Total Steps: 101507, Ep.Steps: 734, Score: -32.90, Avg.Score: -75.29, Time: 01:27:28\n",
      "Ep.: 293, Total Steps: 103107, Ep.Steps: 1600, Score: 246.10, Avg.Score: -72.08, Time: 01:28:55\n",
      "Ep.: 294, Total Steps: 104707, Ep.Steps: 1600, Score: 206.50, Avg.Score: -68.95, Time: 01:30:22\n",
      "Ep.: 295, Total Steps: 106307, Ep.Steps: 1600, Score: 232.71, Avg.Score: -65.70, Time: 01:31:50\n",
      "Ep.: 296, Total Steps: 107907, Ep.Steps: 1600, Score: 225.00, Avg.Score: -62.30, Time: 01:33:14\n",
      "Ep.: 297, Total Steps: 109507, Ep.Steps: 1600, Score: 274.84, Avg.Score: -58.45, Time: 01:34:37\n",
      "Ep.: 298, Total Steps: 111107, Ep.Steps: 1600, Score: 240.41, Avg.Score: -54.95, Time: 01:36:03\n",
      "Ep.: 299, Total Steps: 112675, Ep.Steps: 1568, Score: 285.17, Avg.Score: -51.04, Time: 01:37:31\n",
      "Ep.: 300, Total Steps: 114214, Ep.Steps: 1539, Score: 287.22, Avg.Score: -47.09, Time: 01:38:54\n",
      "Ep.: 301, Total Steps: 115814, Ep.Steps: 1600, Score: 257.79, Avg.Score: -43.39, Time: 01:40:20\n",
      "Ep.: 302, Total Steps: 117402, Ep.Steps: 1588, Score: 286.56, Avg.Score: -39.48, Time: 01:41:45\n",
      "Ep.: 303, Total Steps: 118909, Ep.Steps: 1507, Score: 289.93, Avg.Score: -35.44, Time: 01:43:06\n",
      "Ep.: 304, Total Steps: 120406, Ep.Steps: 1497, Score: 288.71, Avg.Score: -31.49, Time: 01:44:26\n",
      "Ep.: 305, Total Steps: 121833, Ep.Steps: 1427, Score: 288.26, Avg.Score: -27.49, Time: 01:45:42\n",
      "Ep.: 306, Total Steps: 122475, Ep.Steps: 642, Score: -11.91, Avg.Score: -26.55, Time: 01:46:17\n",
      "Ep.: 307, Total Steps: 123349, Ep.Steps: 874, Score: 72.48, Avg.Score: -24.79, Time: 01:47:04\n",
      "Ep.: 308, Total Steps: 123677, Ep.Steps: 328, Score: -48.80, Avg.Score: -24.24, Time: 01:47:21\n",
      "Ep.: 309, Total Steps: 125098, Ep.Steps: 1421, Score: 287.48, Avg.Score: -20.32, Time: 01:48:37\n",
      "Ep.: 310, Total Steps: 125717, Ep.Steps: 619, Score: 18.79, Avg.Score: -19.11, Time: 01:49:11\n",
      "Ep.: 311, Total Steps: 125802, Ep.Steps: 85, Score: -108.88, Avg.Score: -19.22, Time: 01:49:15\n",
      "Ep.: 312, Total Steps: 126780, Ep.Steps: 978, Score: 45.11, Avg.Score: -17.74, Time: 01:50:07\n",
      "Ep.: 313, Total Steps: 128256, Ep.Steps: 1476, Score: 281.50, Avg.Score: -13.88, Time: 01:51:26\n",
      "Ep.: 314, Total Steps: 128373, Ep.Steps: 117, Score: -96.15, Avg.Score: -13.81, Time: 01:51:33\n",
      "Ep.: 315, Total Steps: 128471, Ep.Steps: 98, Score: -115.68, Avg.Score: -13.99, Time: 01:51:38\n",
      "Ep.: 316, Total Steps: 128558, Ep.Steps: 87, Score: -110.55, Avg.Score: -14.07, Time: 01:51:43\n",
      "Ep.: 317, Total Steps: 128652, Ep.Steps: 94, Score: -105.62, Avg.Score: -14.11, Time: 01:51:48\n",
      "Ep.: 318, Total Steps: 128713, Ep.Steps: 61, Score: -111.82, Avg.Score: -14.28, Time: 01:51:51\n",
      "Ep.: 319, Total Steps: 128847, Ep.Steps: 134, Score: -108.45, Avg.Score: -14.38, Time: 01:51:58\n",
      "Ep.: 320, Total Steps: 129055, Ep.Steps: 208, Score: -111.95, Avg.Score: -14.62, Time: 01:52:09\n",
      "Ep.: 321, Total Steps: 129141, Ep.Steps: 86, Score: -102.85, Avg.Score: -14.66, Time: 01:52:14\n",
      "Ep.: 322, Total Steps: 130464, Ep.Steps: 1323, Score: 116.58, Avg.Score: -12.49, Time: 01:53:25\n",
      "Ep.: 323, Total Steps: 130551, Ep.Steps: 87, Score: -100.57, Avg.Score: -12.47, Time: 01:53:29\n",
      "Ep.: 324, Total Steps: 130624, Ep.Steps: 73, Score: -102.28, Avg.Score: -12.52, Time: 01:53:33\n",
      "Ep.: 325, Total Steps: 130691, Ep.Steps: 67, Score: -100.98, Avg.Score: -12.59, Time: 01:53:37\n",
      "Ep.: 326, Total Steps: 130772, Ep.Steps: 81, Score: -99.98, Avg.Score: -12.58, Time: 01:53:42\n",
      "Ep.: 327, Total Steps: 132142, Ep.Steps: 1370, Score: 292.87, Avg.Score: -8.71, Time: 01:54:55\n",
      "Ep.: 328, Total Steps: 132217, Ep.Steps: 75, Score: -107.55, Avg.Score: -8.86, Time: 01:54:59\n",
      "Ep.: 329, Total Steps: 133640, Ep.Steps: 1423, Score: 288.16, Avg.Score: -4.91, Time: 01:56:15\n",
      "Ep.: 330, Total Steps: 135035, Ep.Steps: 1395, Score: 288.31, Avg.Score: -0.98, Time: 01:57:29\n",
      "Ep.: 331, Total Steps: 136370, Ep.Steps: 1335, Score: 291.81, Avg.Score: 3.09, Time: 01:58:41\n",
      "Ep.: 332, Total Steps: 137658, Ep.Steps: 1288, Score: 291.13, Avg.Score: 7.09, Time: 01:59:49\n",
      "Ep.: 333, Total Steps: 138867, Ep.Steps: 1209, Score: 295.00, Avg.Score: 11.02, Time: 02:00:54\n",
      "Ep.: 334, Total Steps: 140065, Ep.Steps: 1198, Score: 294.60, Avg.Score: 15.03, Time: 02:01:58\n",
      "Ep.: 335, Total Steps: 141272, Ep.Steps: 1207, Score: 294.21, Avg.Score: 19.03, Time: 02:03:02\n",
      "Ep.: 336, Total Steps: 142438, Ep.Steps: 1166, Score: 294.83, Avg.Score: 23.02, Time: 02:04:04\n",
      "Ep.: 337, Total Steps: 143568, Ep.Steps: 1130, Score: 165.39, Avg.Score: 25.74, Time: 02:05:05\n",
      "Ep.: 338, Total Steps: 144494, Ep.Steps: 926, Score: 107.13, Avg.Score: 27.80, Time: 02:05:57\n",
      "Ep.: 339, Total Steps: 145718, Ep.Steps: 1224, Score: 291.38, Avg.Score: 31.70, Time: 02:07:03\n",
      "Ep.: 340, Total Steps: 146983, Ep.Steps: 1265, Score: 287.82, Avg.Score: 35.55, Time: 02:08:10\n",
      "Ep.: 341, Total Steps: 147294, Ep.Steps: 311, Score: -41.04, Avg.Score: 36.19, Time: 02:08:26\n",
      "Ep.: 342, Total Steps: 148430, Ep.Steps: 1136, Score: 295.55, Avg.Score: 40.19, Time: 02:09:26\n",
      "Ep.: 343, Total Steps: 149610, Ep.Steps: 1180, Score: 295.64, Avg.Score: 44.20, Time: 02:10:29\n",
      "Ep.: 344, Total Steps: 150835, Ep.Steps: 1225, Score: 292.07, Avg.Score: 48.13, Time: 02:11:33\n",
      "Ep.: 345, Total Steps: 152050, Ep.Steps: 1215, Score: 296.83, Avg.Score: 52.11, Time: 02:12:37\n",
      "Ep.: 346, Total Steps: 153197, Ep.Steps: 1147, Score: 299.89, Avg.Score: 56.14, Time: 02:13:38\n",
      "Ep.: 347, Total Steps: 154410, Ep.Steps: 1213, Score: 294.70, Avg.Score: 60.27, Time: 02:14:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 348, Total Steps: 155556, Ep.Steps: 1146, Score: 297.12, Avg.Score: 64.24, Time: 02:15:43\n",
      "Ep.: 349, Total Steps: 156694, Ep.Steps: 1138, Score: 296.60, Avg.Score: 68.15, Time: 02:16:43\n",
      "Ep.: 350, Total Steps: 157842, Ep.Steps: 1148, Score: 296.71, Avg.Score: 72.06, Time: 02:17:43\n",
      "Ep.: 351, Total Steps: 158888, Ep.Steps: 1046, Score: 301.81, Avg.Score: 76.16, Time: 02:18:38\n",
      "Ep.: 352, Total Steps: 159920, Ep.Steps: 1032, Score: 302.02, Avg.Score: 79.86, Time: 02:19:33\n",
      "Ep.: 353, Total Steps: 161024, Ep.Steps: 1104, Score: 297.86, Avg.Score: 83.62, Time: 02:20:31\n",
      "Ep.: 354, Total Steps: 162081, Ep.Steps: 1057, Score: 301.52, Avg.Score: 87.48, Time: 02:21:27\n",
      "Ep.: 355, Total Steps: 163173, Ep.Steps: 1092, Score: 298.22, Avg.Score: 91.27, Time: 02:22:24\n",
      "Ep.: 356, Total Steps: 163859, Ep.Steps: 686, Score: 79.78, Avg.Score: 92.73, Time: 02:23:01\n",
      "Ep.: 357, Total Steps: 164907, Ep.Steps: 1048, Score: 301.53, Avg.Score: 96.37, Time: 02:23:56\n",
      "Ep.: 358, Total Steps: 165953, Ep.Steps: 1046, Score: 302.69, Avg.Score: 99.81, Time: 02:24:51\n",
      "Ep.: 359, Total Steps: 166982, Ep.Steps: 1029, Score: 301.53, Avg.Score: 103.27, Time: 02:25:46\n",
      "Ep.: 360, Total Steps: 168046, Ep.Steps: 1064, Score: 300.54, Avg.Score: 106.68, Time: 02:26:42\n",
      "Ep.: 361, Total Steps: 169080, Ep.Steps: 1034, Score: 301.72, Avg.Score: 109.86, Time: 02:27:36\n",
      "Ep.: 362, Total Steps: 170071, Ep.Steps: 991, Score: 305.10, Avg.Score: 114.08, Time: 02:28:29\n",
      "Ep.: 363, Total Steps: 171167, Ep.Steps: 1096, Score: 299.15, Avg.Score: 117.87, Time: 02:29:26\n",
      "Ep.: 364, Total Steps: 172209, Ep.Steps: 1042, Score: 305.83, Avg.Score: 121.50, Time: 02:30:21\n",
      "Ep.: 365, Total Steps: 173245, Ep.Steps: 1036, Score: 302.47, Avg.Score: 124.97, Time: 02:31:16\n",
      "Ep.: 366, Total Steps: 173449, Ep.Steps: 204, Score: -60.13, Avg.Score: 124.77, Time: 02:31:27\n",
      "Ep.: 367, Total Steps: 174418, Ep.Steps: 969, Score: 306.59, Avg.Score: 128.68, Time: 02:32:18\n",
      "Ep.: 368, Total Steps: 175404, Ep.Steps: 986, Score: 304.95, Avg.Score: 131.56, Time: 02:33:10\n",
      "Ep.: 369, Total Steps: 176462, Ep.Steps: 1058, Score: 302.81, Avg.Score: 135.42, Time: 02:34:06\n",
      "Ep.: 370, Total Steps: 177466, Ep.Steps: 1004, Score: 302.99, Avg.Score: 139.30, Time: 02:34:59\n",
      "Ep.: 371, Total Steps: 178477, Ep.Steps: 1011, Score: 302.07, Avg.Score: 143.13, Time: 02:35:52\n",
      "Ep.: 372, Total Steps: 179479, Ep.Steps: 1002, Score: 302.33, Avg.Score: 147.17, Time: 02:36:48\n",
      "Ep.: 373, Total Steps: 180515, Ep.Steps: 1036, Score: 298.41, Avg.Score: 151.11, Time: 02:37:43\n",
      "Ep.: 374, Total Steps: 181506, Ep.Steps: 991, Score: 301.67, Avg.Score: 154.84, Time: 02:38:35\n",
      "Ep.: 375, Total Steps: 182503, Ep.Steps: 997, Score: 301.72, Avg.Score: 158.77, Time: 02:39:27\n",
      "Ep.: 376, Total Steps: 183450, Ep.Steps: 947, Score: 302.74, Avg.Score: 161.98, Time: 02:40:17\n",
      "Ep.: 377, Total Steps: 184414, Ep.Steps: 964, Score: 302.30, Avg.Score: 165.96, Time: 02:41:08\n",
      "Ep.: 378, Total Steps: 185377, Ep.Steps: 963, Score: 301.89, Avg.Score: 169.50, Time: 02:41:58\n",
      "Ep.: 379, Total Steps: 186333, Ep.Steps: 956, Score: 302.13, Avg.Score: 172.38, Time: 02:42:49\n",
      "Ep.: 380, Total Steps: 187318, Ep.Steps: 985, Score: 302.97, Avg.Score: 174.80, Time: 02:43:40\n",
      "Ep.: 381, Total Steps: 188294, Ep.Steps: 976, Score: 302.13, Avg.Score: 176.72, Time: 02:44:32\n",
      "Ep.: 382, Total Steps: 189285, Ep.Steps: 991, Score: 302.34, Avg.Score: 178.51, Time: 02:45:24\n",
      "Ep.: 383, Total Steps: 190281, Ep.Steps: 996, Score: 301.14, Avg.Score: 181.93, Time: 02:46:16\n",
      "Ep.: 384, Total Steps: 191274, Ep.Steps: 993, Score: 303.26, Avg.Score: 183.40, Time: 02:47:09\n",
      "Ep.: 385, Total Steps: 192284, Ep.Steps: 1010, Score: 301.76, Avg.Score: 187.53, Time: 02:48:02\n",
      "Ep.: 386, Total Steps: 193296, Ep.Steps: 1012, Score: 301.66, Avg.Score: 191.78, Time: 02:48:55\n",
      "Ep.: 387, Total Steps: 194260, Ep.Steps: 964, Score: 303.64, Avg.Score: 195.88, Time: 02:49:45\n",
      "Ep.: 388, Total Steps: 194368, Ep.Steps: 108, Score: -121.97, Avg.Score: 195.90, Time: 02:49:51\n",
      "Ep.: 389, Total Steps: 195387, Ep.Steps: 1019, Score: 302.97, Avg.Score: 197.29, Time: 02:50:45\n",
      "Ep.: 390, Total Steps: 196371, Ep.Steps: 984, Score: 302.45, Avg.Score: 198.55, Time: 02:51:37\n",
      "Ep.: 391, Total Steps: 196868, Ep.Steps: 497, Score: 18.49, Avg.Score: 198.56, Time: 02:52:03\n",
      "Ep.: 392, Total Steps: 196960, Ep.Steps: 92, Score: -118.37, Avg.Score: 197.71, Time: 02:52:07\n",
      "Ep.: 393, Total Steps: 197402, Ep.Steps: 442, Score: -23.55, Avg.Score: 195.01, Time: 02:52:31\n",
      "Ep.: 394, Total Steps: 198472, Ep.Steps: 1070, Score: 298.17, Avg.Score: 195.93, Time: 02:53:27\n",
      "Ep.: 395, Total Steps: 199527, Ep.Steps: 1055, Score: 298.37, Avg.Score: 196.58, Time: 02:54:23\n",
      "Ep.: 396, Total Steps: 200629, Ep.Steps: 1102, Score: 297.52, Avg.Score: 197.31, Time: 02:55:21\n",
      "Ep.: 397, Total Steps: 200716, Ep.Steps: 87, Score: -122.14, Avg.Score: 193.34, Time: 02:55:26\n",
      "Ep.: 398, Total Steps: 201746, Ep.Steps: 1030, Score: 300.02, Avg.Score: 193.93, Time: 02:56:20\n",
      "Ep.: 399, Total Steps: 202788, Ep.Steps: 1042, Score: 298.02, Avg.Score: 194.06, Time: 02:57:15\n",
      "Ep.: 400, Total Steps: 203870, Ep.Steps: 1082, Score: 297.46, Avg.Score: 194.17, Time: 02:58:11\n",
      "Ep.: 401, Total Steps: 203977, Ep.Steps: 107, Score: -100.15, Avg.Score: 190.59, Time: 02:58:17\n",
      "Ep.: 402, Total Steps: 205020, Ep.Steps: 1043, Score: 300.32, Avg.Score: 190.72, Time: 02:59:12\n",
      "Ep.: 403, Total Steps: 206063, Ep.Steps: 1043, Score: 301.21, Avg.Score: 190.84, Time: 03:00:07\n",
      "Ep.: 404, Total Steps: 207067, Ep.Steps: 1004, Score: 304.25, Avg.Score: 190.99, Time: 03:01:00\n",
      "Ep.: 405, Total Steps: 208016, Ep.Steps: 949, Score: 306.79, Avg.Score: 191.18, Time: 03:01:50\n",
      "Ep.: 406, Total Steps: 209008, Ep.Steps: 992, Score: 305.41, Avg.Score: 194.35, Time: 03:02:42\n",
      "Ep.: 407, Total Steps: 210016, Ep.Steps: 1008, Score: 303.56, Avg.Score: 196.66, Time: 03:03:35\n",
      "Ep.: 408, Total Steps: 211008, Ep.Steps: 992, Score: 303.60, Avg.Score: 200.18, Time: 03:04:27\n",
      "Ep.: 409, Total Steps: 211986, Ep.Steps: 978, Score: 303.35, Avg.Score: 200.34, Time: 03:05:18\n",
      "Ep.: 410, Total Steps: 212955, Ep.Steps: 969, Score: 305.06, Avg.Score: 203.21, Time: 03:06:13\n",
      "Ep.: 411, Total Steps: 213890, Ep.Steps: 935, Score: 306.45, Avg.Score: 207.36, Time: 03:07:04\n",
      "Ep.: 412, Total Steps: 214843, Ep.Steps: 953, Score: 305.82, Avg.Score: 209.97, Time: 03:07:56\n",
      "Ep.: 413, Total Steps: 215774, Ep.Steps: 931, Score: 305.52, Avg.Score: 210.21, Time: 03:08:47\n",
      "Ep.: 414, Total Steps: 216081, Ep.Steps: 307, Score: -15.46, Avg.Score: 211.01, Time: 03:09:03\n",
      "Ep.: 415, Total Steps: 216471, Ep.Steps: 390, Score: -0.23, Avg.Score: 212.17, Time: 03:09:24\n",
      "Ep.: 416, Total Steps: 217411, Ep.Steps: 940, Score: 306.28, Avg.Score: 216.34, Time: 03:10:15\n",
      "Ep.: 417, Total Steps: 218352, Ep.Steps: 941, Score: 306.67, Avg.Score: 220.46, Time: 03:11:06\n",
      "Ep.: 418, Total Steps: 219305, Ep.Steps: 953, Score: 306.73, Avg.Score: 224.64, Time: 03:11:57\n",
      "Ep.: 419, Total Steps: 220290, Ep.Steps: 985, Score: 306.80, Avg.Score: 228.80, Time: 03:12:50\n",
      "Ep.: 420, Total Steps: 221255, Ep.Steps: 965, Score: 304.97, Avg.Score: 232.97, Time: 03:13:42\n",
      "Ep.: 421, Total Steps: 222142, Ep.Steps: 887, Score: 307.06, Avg.Score: 237.07, Time: 03:14:30\n",
      "Ep.: 422, Total Steps: 223071, Ep.Steps: 929, Score: 308.10, Avg.Score: 238.98, Time: 03:15:21\n",
      "Ep.: 423, Total Steps: 224007, Ep.Steps: 936, Score: 307.33, Avg.Score: 243.06, Time: 03:16:11\n",
      "Ep.: 424, Total Steps: 224927, Ep.Steps: 920, Score: 307.84, Avg.Score: 247.16, Time: 03:17:01\n",
      "Ep.: 425, Total Steps: 225046, Ep.Steps: 119, Score: -88.63, Avg.Score: 247.28, Time: 03:17:07\n",
      "Ep.: 426, Total Steps: 225991, Ep.Steps: 945, Score: 308.59, Avg.Score: 251.37, Time: 03:17:58\n",
      "Ep.: 427, Total Steps: 226930, Ep.Steps: 939, Score: 308.59, Avg.Score: 251.53, Time: 03:18:49\n",
      "Ep.: 428, Total Steps: 227850, Ep.Steps: 920, Score: 309.74, Avg.Score: 255.70, Time: 03:19:38\n",
      "Ep.: 429, Total Steps: 228791, Ep.Steps: 941, Score: 308.51, Avg.Score: 255.90, Time: 03:20:29\n",
      "Ep.: 430, Total Steps: 229755, Ep.Steps: 964, Score: 306.57, Avg.Score: 256.09, Time: 03:21:21\n",
      "Ep.: 431, Total Steps: 230706, Ep.Steps: 951, Score: 306.74, Avg.Score: 256.24, Time: 03:22:12\n",
      "Ep.: 432, Total Steps: 231681, Ep.Steps: 975, Score: 305.96, Avg.Score: 256.38, Time: 03:23:06\n",
      "Ep.: 433, Total Steps: 232625, Ep.Steps: 944, Score: 307.01, Avg.Score: 256.50, Time: 03:23:57\n",
      "Ep.: 434, Total Steps: 233563, Ep.Steps: 938, Score: 307.05, Avg.Score: 256.63, Time: 03:24:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 435, Total Steps: 234532, Ep.Steps: 969, Score: 305.93, Avg.Score: 256.75, Time: 03:25:40\n",
      "Ep.: 436, Total Steps: 235503, Ep.Steps: 971, Score: 301.91, Avg.Score: 256.82, Time: 03:26:32\n",
      "Ep.: 437, Total Steps: 235568, Ep.Steps: 65, Score: -112.04, Avg.Score: 254.04, Time: 03:26:36\n",
      "Ep.: 438, Total Steps: 236573, Ep.Steps: 1005, Score: 304.96, Avg.Score: 256.02, Time: 03:27:30\n",
      "Ep.: 439, Total Steps: 236925, Ep.Steps: 352, Score: 6.82, Avg.Score: 253.17, Time: 03:27:49\n",
      "Ep.: 440, Total Steps: 237854, Ep.Steps: 929, Score: 308.97, Avg.Score: 253.39, Time: 03:28:39\n",
      "Ep.: 441, Total Steps: 238818, Ep.Steps: 964, Score: 305.04, Avg.Score: 256.85, Time: 03:29:31\n",
      "Ep.: 442, Total Steps: 239724, Ep.Steps: 906, Score: 177.16, Avg.Score: 255.66, Time: 03:30:20\n",
      "Ep.: 443, Total Steps: 240671, Ep.Steps: 947, Score: 308.66, Avg.Score: 255.79, Time: 03:31:11\n",
      "Ep.: 444, Total Steps: 241605, Ep.Steps: 934, Score: 309.11, Avg.Score: 255.96, Time: 03:32:02\n",
      "Ep.: 445, Total Steps: 242533, Ep.Steps: 928, Score: 307.05, Avg.Score: 256.07, Time: 03:32:52\n",
      "Ep.: 446, Total Steps: 243455, Ep.Steps: 922, Score: 307.83, Avg.Score: 256.15, Time: 03:33:41\n",
      "Ep.: 447, Total Steps: 244359, Ep.Steps: 904, Score: 307.71, Avg.Score: 256.28, Time: 03:34:30\n",
      "Ep.: 448, Total Steps: 245259, Ep.Steps: 900, Score: 308.64, Avg.Score: 256.39, Time: 03:35:19\n",
      "Ep.: 449, Total Steps: 246145, Ep.Steps: 886, Score: 309.74, Avg.Score: 256.52, Time: 03:36:08\n",
      "Ep.: 450, Total Steps: 247025, Ep.Steps: 880, Score: 309.86, Avg.Score: 256.65, Time: 03:36:55\n",
      "Ep.: 451, Total Steps: 247904, Ep.Steps: 879, Score: 310.29, Avg.Score: 256.74, Time: 03:37:40\n",
      "Ep.: 452, Total Steps: 248796, Ep.Steps: 892, Score: 310.19, Avg.Score: 256.82, Time: 03:38:28\n",
      "Ep.: 453, Total Steps: 249661, Ep.Steps: 865, Score: 310.16, Avg.Score: 256.94, Time: 03:39:13\n",
      "Ep.: 454, Total Steps: 250545, Ep.Steps: 884, Score: 310.44, Avg.Score: 257.03, Time: 03:40:00\n",
      "Ep.: 455, Total Steps: 251433, Ep.Steps: 888, Score: 310.28, Avg.Score: 257.15, Time: 03:40:46\n",
      "Ep.: 456, Total Steps: 252265, Ep.Steps: 832, Score: 312.66, Avg.Score: 259.48, Time: 03:41:30\n",
      "Ep.: 457, Total Steps: 253136, Ep.Steps: 871, Score: 310.23, Avg.Score: 259.57, Time: 03:42:15\n",
      "Ep.: 458, Total Steps: 254030, Ep.Steps: 894, Score: 311.76, Avg.Score: 259.66, Time: 03:43:02\n",
      "Ep.: 459, Total Steps: 254865, Ep.Steps: 835, Score: 312.69, Avg.Score: 259.77, Time: 03:43:45\n",
      "Ep.: 460, Total Steps: 255723, Ep.Steps: 858, Score: 312.26, Avg.Score: 259.89, Time: 03:44:30\n",
      "Ep.: 461, Total Steps: 256585, Ep.Steps: 862, Score: 309.79, Avg.Score: 259.97, Time: 03:45:15\n",
      "Ep.: 462, Total Steps: 257482, Ep.Steps: 897, Score: 311.41, Avg.Score: 260.03, Time: 03:46:02\n",
      "Ep.: 463, Total Steps: 258367, Ep.Steps: 885, Score: 311.43, Avg.Score: 260.15, Time: 03:46:48\n",
      "Ep.: 464, Total Steps: 259236, Ep.Steps: 869, Score: 309.74, Avg.Score: 260.19, Time: 03:47:33\n",
      "Ep.: 465, Total Steps: 260074, Ep.Steps: 838, Score: 312.95, Avg.Score: 260.30, Time: 03:48:17\n",
      "Ep.: 466, Total Steps: 260931, Ep.Steps: 857, Score: 312.34, Avg.Score: 264.02, Time: 03:49:02\n",
      "Ep.: 467, Total Steps: 261792, Ep.Steps: 861, Score: 310.91, Avg.Score: 264.07, Time: 03:49:46\n",
      "Ep.: 468, Total Steps: 262663, Ep.Steps: 871, Score: 312.82, Avg.Score: 264.15, Time: 03:50:32\n",
      "Ep.: 469, Total Steps: 263506, Ep.Steps: 843, Score: 312.07, Avg.Score: 264.24, Time: 03:51:16\n",
      "Ep.: 470, Total Steps: 264379, Ep.Steps: 873, Score: 311.08, Avg.Score: 264.32, Time: 03:52:01\n",
      "Ep.: 471, Total Steps: 265207, Ep.Steps: 828, Score: 312.35, Avg.Score: 264.42, Time: 03:52:44\n",
      "Ep.: 472, Total Steps: 266100, Ep.Steps: 893, Score: 309.75, Avg.Score: 264.50, Time: 03:53:31\n",
      "Ep.: 473, Total Steps: 266957, Ep.Steps: 857, Score: 314.09, Avg.Score: 264.65, Time: 03:54:15\n",
      "Ep.: 474, Total Steps: 267802, Ep.Steps: 845, Score: 314.01, Avg.Score: 264.78, Time: 03:54:59\n",
      "Ep.: 475, Total Steps: 268628, Ep.Steps: 826, Score: 313.25, Avg.Score: 264.89, Time: 03:55:42\n",
      "Ep.: 476, Total Steps: 269491, Ep.Steps: 863, Score: 313.81, Avg.Score: 265.00, Time: 03:56:27\n",
      "Ep.: 477, Total Steps: 270131, Ep.Steps: 640, Score: 126.89, Avg.Score: 263.25, Time: 03:57:01\n",
      "Ep.: 478, Total Steps: 270975, Ep.Steps: 844, Score: 312.84, Avg.Score: 263.36, Time: 03:57:45\n",
      "Ep.: 479, Total Steps: 271821, Ep.Steps: 846, Score: 312.64, Avg.Score: 263.46, Time: 03:58:29\n",
      "Ep.: 480, Total Steps: 272690, Ep.Steps: 869, Score: 312.48, Avg.Score: 263.56, Time: 03:59:14\n",
      "Ep.: 481, Total Steps: 273516, Ep.Steps: 826, Score: 313.12, Avg.Score: 263.67, Time: 03:59:57\n",
      "Ep.: 482, Total Steps: 274332, Ep.Steps: 816, Score: 312.88, Avg.Score: 263.77, Time: 04:00:40\n",
      "Ep.: 483, Total Steps: 275190, Ep.Steps: 858, Score: 312.16, Avg.Score: 263.88, Time: 04:01:24\n",
      "Ep.: 484, Total Steps: 276015, Ep.Steps: 825, Score: 314.07, Avg.Score: 263.99, Time: 04:02:07\n",
      "Ep.: 485, Total Steps: 276865, Ep.Steps: 850, Score: 313.98, Avg.Score: 264.11, Time: 04:02:52\n",
      "Ep.: 486, Total Steps: 277739, Ep.Steps: 874, Score: 311.76, Avg.Score: 264.21, Time: 04:03:37\n",
      "Ep.: 487, Total Steps: 278576, Ep.Steps: 837, Score: 312.65, Avg.Score: 264.30, Time: 04:04:21\n",
      "Ep.: 488, Total Steps: 279441, Ep.Steps: 865, Score: 312.47, Avg.Score: 268.65, Time: 04:05:06\n",
      "Ep.: 489, Total Steps: 280326, Ep.Steps: 885, Score: 312.23, Avg.Score: 268.74, Time: 04:05:52\n",
      "Ep.: 490, Total Steps: 281165, Ep.Steps: 839, Score: 313.86, Avg.Score: 268.86, Time: 04:06:38\n",
      "Ep.: 491, Total Steps: 281733, Ep.Steps: 568, Score: 96.62, Avg.Score: 269.64, Time: 04:07:08\n",
      "Ep.: 492, Total Steps: 282575, Ep.Steps: 842, Score: 314.41, Avg.Score: 273.96, Time: 04:07:52\n",
      "Ep.: 493, Total Steps: 283422, Ep.Steps: 847, Score: 313.62, Avg.Score: 277.34, Time: 04:08:37\n",
      "Ep.: 494, Total Steps: 284245, Ep.Steps: 823, Score: 313.74, Avg.Score: 277.49, Time: 04:09:20\n",
      "Ep.: 495, Total Steps: 285098, Ep.Steps: 853, Score: 311.36, Avg.Score: 277.62, Time: 04:10:05\n",
      "Ep.: 496, Total Steps: 285929, Ep.Steps: 831, Score: 310.66, Avg.Score: 277.75, Time: 04:10:48\n",
      "Ep.: 497, Total Steps: 286812, Ep.Steps: 883, Score: 310.23, Avg.Score: 282.08, Time: 04:11:34\n",
      "Ep.: 498, Total Steps: 287645, Ep.Steps: 833, Score: 311.74, Avg.Score: 282.19, Time: 04:12:18\n",
      "Ep.: 499, Total Steps: 288498, Ep.Steps: 853, Score: 312.00, Avg.Score: 282.33, Time: 04:13:02\n",
      "Ep.: 500, Total Steps: 289332, Ep.Steps: 834, Score: 310.59, Avg.Score: 282.47, Time: 04:13:46\n",
      "Ep.: 501, Total Steps: 290162, Ep.Steps: 830, Score: 314.34, Avg.Score: 286.61, Time: 04:14:29\n",
      "Ep.: 502, Total Steps: 290995, Ep.Steps: 833, Score: 314.88, Avg.Score: 286.76, Time: 04:15:12\n",
      "Ep.: 503, Total Steps: 291847, Ep.Steps: 852, Score: 311.76, Avg.Score: 286.86, Time: 04:15:57\n",
      "Ep.: 504, Total Steps: 292682, Ep.Steps: 835, Score: 312.70, Avg.Score: 286.95, Time: 04:16:41\n",
      "Ep.: 505, Total Steps: 293490, Ep.Steps: 808, Score: 315.05, Avg.Score: 287.03, Time: 04:17:23\n",
      "Ep.: 506, Total Steps: 294314, Ep.Steps: 824, Score: 313.02, Avg.Score: 287.10, Time: 04:18:06\n",
      "Ep.: 507, Total Steps: 295156, Ep.Steps: 842, Score: 312.10, Avg.Score: 287.19, Time: 04:18:50\n",
      "Ep.: 508, Total Steps: 295991, Ep.Steps: 835, Score: 313.57, Avg.Score: 287.29, Time: 04:19:33\n",
      "Ep.: 509, Total Steps: 296810, Ep.Steps: 819, Score: 315.28, Avg.Score: 287.41, Time: 04:20:16\n",
      "Ep.: 510, Total Steps: 297116, Ep.Steps: 306, Score: 5.58, Avg.Score: 284.41, Time: 04:20:32\n",
      "Ep.: 511, Total Steps: 297952, Ep.Steps: 836, Score: 315.91, Avg.Score: 284.51, Time: 04:21:15\n",
      "Ep.: 512, Total Steps: 298774, Ep.Steps: 822, Score: 315.66, Avg.Score: 284.61, Time: 04:21:58\n",
      "Ep.: 513, Total Steps: 299603, Ep.Steps: 829, Score: 315.84, Avg.Score: 284.71, Time: 04:22:41\n",
      "Ep.: 514, Total Steps: 300432, Ep.Steps: 829, Score: 315.47, Avg.Score: 288.02, Time: 04:23:24\n",
      "Ep.: 515, Total Steps: 301242, Ep.Steps: 810, Score: 313.95, Avg.Score: 291.16, Time: 04:24:07\n",
      "Ep.: 516, Total Steps: 302073, Ep.Steps: 831, Score: 312.88, Avg.Score: 291.23, Time: 04:24:50\n",
      "Ep.: 517, Total Steps: 302901, Ep.Steps: 828, Score: 312.74, Avg.Score: 291.29, Time: 04:25:33\n",
      "Ep.: 518, Total Steps: 303715, Ep.Steps: 814, Score: 314.86, Avg.Score: 291.37, Time: 04:26:15\n",
      "Ep.: 519, Total Steps: 304531, Ep.Steps: 816, Score: 315.96, Avg.Score: 291.46, Time: 04:26:58\n",
      "Ep.: 520, Total Steps: 305358, Ep.Steps: 827, Score: 314.16, Avg.Score: 291.55, Time: 04:27:41\n",
      "Ep.: 521, Total Steps: 306177, Ep.Steps: 819, Score: 312.65, Avg.Score: 291.61, Time: 04:28:24\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 522, Total Steps: 306427, Ep.Steps: 250, Score: -24.15, Avg.Score: 288.29, Time: 04:28:37\n",
      "Ep.: 523, Total Steps: 307247, Ep.Steps: 820, Score: 314.58, Avg.Score: 288.36, Time: 04:29:20\n",
      "Ep.: 524, Total Steps: 308084, Ep.Steps: 837, Score: 313.28, Avg.Score: 288.41, Time: 04:30:04\n",
      "Ep.: 525, Total Steps: 308911, Ep.Steps: 827, Score: 314.60, Avg.Score: 292.45, Time: 04:30:47\n",
      "Ep.: 526, Total Steps: 309743, Ep.Steps: 832, Score: 314.31, Avg.Score: 292.50, Time: 04:31:30\n",
      "Ep.: 527, Total Steps: 310582, Ep.Steps: 839, Score: 316.50, Avg.Score: 292.58, Time: 04:32:14\n",
      "Ep.: 528, Total Steps: 311438, Ep.Steps: 856, Score: 314.36, Avg.Score: 292.63, Time: 04:32:58\n",
      "Ep.: 529, Total Steps: 312282, Ep.Steps: 844, Score: 315.17, Avg.Score: 292.69, Time: 04:33:42\n",
      "Ep.: 530, Total Steps: 313122, Ep.Steps: 840, Score: 315.30, Avg.Score: 292.78, Time: 04:34:26\n",
      "Ep.: 531, Total Steps: 313956, Ep.Steps: 834, Score: 314.62, Avg.Score: 292.86, Time: 04:35:09\n",
      "Ep.: 532, Total Steps: 314831, Ep.Steps: 875, Score: 312.64, Avg.Score: 292.93, Time: 04:35:55\n",
      "Ep.: 533, Total Steps: 315660, Ep.Steps: 829, Score: 313.58, Avg.Score: 292.99, Time: 04:36:42\n",
      "Ep.: 534, Total Steps: 316460, Ep.Steps: 800, Score: 316.48, Avg.Score: 293.09, Time: 04:37:24\n",
      "Ep.: 535, Total Steps: 317257, Ep.Steps: 797, Score: 314.95, Avg.Score: 293.18, Time: 04:38:06\n",
      "Ep.: 536, Total Steps: 318063, Ep.Steps: 806, Score: 315.93, Avg.Score: 293.32, Time: 04:38:48\n",
      "Ep.: 537, Total Steps: 318861, Ep.Steps: 798, Score: 313.93, Avg.Score: 297.58, Time: 04:39:30\n",
      "Ep.: 538, Total Steps: 319687, Ep.Steps: 826, Score: 315.65, Avg.Score: 297.68, Time: 04:40:13\n",
      "Ep.: 539, Total Steps: 320497, Ep.Steps: 810, Score: 313.91, Avg.Score: 300.76, Time: 04:40:55\n",
      "Solved environment with Avg Score:   300.7556449074799\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from  collections  import deque\n",
    "\n",
    "def sac_train(max_steps):\n",
    "\n",
    "    total_numsteps = 0\n",
    "    updates = 0\n",
    "    num_episodes = 10001\n",
    "    updates=0\n",
    "\n",
    "    time_start = time.time()\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores_array = []\n",
    "    avg_scores_array = [] \n",
    "    \n",
    "    for i_episode in range(num_episodes): \n",
    "        episode_reward = 0\n",
    "        episode_steps = 0\n",
    "        done = False\n",
    "        state = env.reset()\n",
    "\n",
    "        for step in range(max_steps):    \n",
    "            if start_steps > total_numsteps:\n",
    "                action = env.action_space.sample()  # Sample random action\n",
    "            else:\n",
    "                action = agent.select_action(state)  # Sample action from policy\n",
    "\n",
    "            if len(memory) > batch_size:\n",
    "                \n",
    "                # Update parameters of all the networks\n",
    "                agent.update_parameters(memory, batch_size, updates)\n",
    "\n",
    "                updates += 1\n",
    "\n",
    "            next_state, reward, done, _ = env.step(action) # Step\n",
    "            episode_steps += 1\n",
    "            total_numsteps += 1\n",
    "            episode_reward += reward\n",
    "\n",
    "            mask = 1 if episode_steps == env._max_episode_steps else float(not done)\n",
    "\n",
    "            memory.push(state, action, reward, next_state, mask) # Append transition to memory\n",
    "\n",
    "            state = next_state\n",
    "            \n",
    "            if done:\n",
    "                break\n",
    "\n",
    "        scores_deque.append(episode_reward)\n",
    "        scores_array.append(episode_reward)        \n",
    "        avg_score = np.mean(scores_deque)\n",
    "        avg_scores_array.append(avg_score)\n",
    "        \n",
    "        if i_episode % 20 == 0 and i_episode > 0:\n",
    "            save(agent, 'dir_chk_lr0005', 'weights', str(i_episode))\n",
    "\n",
    "        s =  (int)(time.time() - time_start)\n",
    "            \n",
    "        print(\"Ep.: {}, Total Steps: {}, Ep.Steps: {}, Score: {:.2f}, Avg.Score: {:.2f}, Time: {:02}:{:02}:{:02}\".\\\n",
    "            format(i_episode, total_numsteps, episode_steps, episode_reward, avg_score, \\\n",
    "                  s//3600, s%3600//60, s%60))\n",
    "\n",
    "                    \n",
    "        if (avg_score > 300.5):\n",
    "            print('Solved environment with Avg Score:  ', avg_score)\n",
    "            break;\n",
    "            \n",
    "    return scores_array, avg_scores_array \n",
    "\n",
    "scores, avg_scores = sac_train(max_steps=max_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(agent, 'dir_chk_lr0005', 'weights', 'final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of scores:  540 , len of avg_scores:  540\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAEGCAYAAACtn3UnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3hcxfm279miZlmWZcu9d8sFMKbb2KYbDKaZkoSSkNCSEEj7pRKSkOCE8JFAQg0JNaEYA6GDATe6e++2cJMlS7J62TLfH1u05Zzds9LK2pXeOxfR7pw5c2ZlS/P4rUprjSAIgiAIQrpg6+gNCIIgCIIgJIKIF0EQBEEQ0goRL4IgCIIgpBUiXgRBEARBSCtEvAiCIAiCkFY4OnoD7Unv3r31sGHDOnobgiAIacXKlSsPa60LO3ofgmBGpxYvw4YNY8WKFR29DUEQhLRCKVXc0XsQhFiI20gQBEEQhLRCxIsgCIIgCGmFiBdBEARBENKKTh3zIgiCIKQOK1eu7ONwOP4JTET+8SyY4wU2uN3ubx9//PGlRhNEvAiCIAhHBYfD8c9+/fqNLywsrLTZbNJYTzDE6/WqsrKyopKSkn8CFxnNEeUrCIIgHC0mFhYWVotwEWJhs9l0YWFhFT4LnfGco7gfQRAEoWtjE+EiWMH/98RUo4h4EQRBSDEaXR427K9i26EavF7fWb+rrJYmtyc4p6SqkbfWHwy+33+kgdfXHqC4vI5GlydqTUHoTEjMiyAIQht4b2MJ5XXNjCzM5cThBQBordlSUsPmg9VceMwAPF7Nqq8qWf3VEcb27c6MsYU47TZcHi9/eHMzAJX1zZwyohdLtpXx9oaS4PqzxhZy7oR+/Gzhes4c14eLjh3Aa2sO8OGWljjGKUPy+aqigcO1TcGxR75xPOdN7HeUvgvpxf/93//1e/nll3vZbDZts9l46KGHis8444y6jt6XYB0RL4IgCAb8b+0B3lh7ALtNkeW0M2/qIFYVV7Jw1X4q6puZPCgfl9vLp7vKg/c47YoB+dkUl9cHx3744tqotW87YxSnjOzNfe9tZUVxZXD8tTUHouY98OEOPtpaBsAHW0r5YEt48sWZ4/pEjQHc//42zinqi82mWvcN6KQsWrSo27vvvpu/fv36TdnZ2frgwYOOpqamVn+TXC4XTqczmVsULCDiRRCETsmO0hoWby3jmlOGkumwh11rdHlodHlw2G18VV5P0YA81uw9wgtf7mVrSTUeDWv3Hgm755XV+8PeL91WFnw9fXRvlm0/jMujyclwcOmUgewsrcVuU+w+XEdlvSvs3gc+3MEDH+4Ivu+dm0HRgB7YFCz2C5VMh40fnjOWogE9WLKtjFtnjuSyhz+htMZnXdny+/PIcvo+11vrD7KjtJabZ4wEoLSmkZwMhwgXA/bv3+8sKChwZ2dna4D+/fu7AZYsWZJz++23D6mvr7dlZGTopUuXbs3MzNTXXnvt0HXr1uXY7Xb+/Oc/773wwgtrHnjggV5vv/12j6amJlt9fb3ts88+2/brX/+67yuvvFLQ3NysLrjggiP333//gdg7EdqCiBdBEJJKbZObDLuNDEfyQuq01uytaGBIrxwAGpo9ZDps2GyKqnoX20tr6J7l5PFlu9hf2cCRBhebD1YD8MHmUn41Zzw7SmuZPrqQgm4Z3PLsSj7aWsbw3t3YfbiO7lkO6ps9OO2KyQPzybQpzinqi9ur+fb04by9voSdZbUcqXdxy8yRZDntjOqTyz+X7WL34Tq+NW04y7YfBnzWktmT+gPg9WqUgpdW7uOnC9YB8MwNJ7K1pIbNB2uYNa6QnjkZnDKiV1BofP+/q3l97QGUX3ecN7Ff0P3j8M85dWSvoHABON//vACDeuYk7XvfXvxkwdrB20pqkrrRMf261997+TF7Y825+OKLq++5554Bw4YNmzht2rTqq6++uuLMM8+s+/rXvz7yueee2zljxoz6iooKW25urvfuu+/uC7Bt27ZNq1evzjr//PNH79y5cwPAqlWrctetW7exb9++noULF+bt2LEja926dZu11px11lmj3n777dzZs2fXJvPzCS10mHhRSmUBS4FM/z4WaK1/o5QaDjwPFACrgGu01s1KqUzgaeB4oBy4Umu9p0M2LwhCGOv3VfHyqn1U1jez8UA1xeV1bPzteUkTMI8t3cU9b29h/qWTmDK0J5c9/Am9umVw4+kj+d0bG2l0eQ3vm3f8IF5auY8LHlgOwNBeOTx49XFBN8zuw74wB5tSXHTMAO66cAI9cqJdAKeO7G24/h8umQSEW2GUarF2BASJLWQsJ8PBt6ePMP2sKvg12moSWFuJQaXV9OjRw7thw4ZN77zzTvcPPvig+3XXXTfyjjvuONinTx/XjBkz6gEKCgq8AJ988knu97///VKA4447rnHAgAHN69evzwKYPn16dd++fT0A77zzTt7SpUvzioqKigDq6+ttW7ZsyRLx0n50pOWlCThDa12rlHICy5VSbwM/BO7XWj+vlHoEuAF42P+1Ums9Sil1FfAn4MqO2rwgdEUaXR4q65vp3yMb8AWrDu/djQv/vjxq7uWPfMJ3po/gjHF9WLv3CGW1TeRkONh4oIpx/bqzZNthspw2NuyvYvroQi6dMtDQYtDo8vDIkp0A/Gzh+uB4TaObpz/dQ6PLy9UnDmZV8RHOGN+HvRX1vLHuIJkOG/Mvm8w5E/qxZm8ldqV44MMdXPT3j8PWv+bkofzmwiIc9tYLrVBxYiQsVNjc2GsF7o8lUIyETboRz0LSnjgcDubMmVMzZ86cmsmTJzc88sgjhUqpqBRurc2zunNycryh826//faDP/nJTw6305aFCDpMvGjf34qAKnX6/9PAGcDX/ONPAXfhEy9z/a8BFgB/V0opHetvlyAIbaaqwYXb46VXbiYXPLCMnWU+a8V3pg/n8WW7Te9bt6+K7/93taVnfLmnkvX7q3j82qkAuD1eth2qpWhAHn/7YDuV9S4evPo4vthdQUV9M7fMGMmtz60i8NM/Y0wf/njJJJRSYc+02xRnF/Xl7KK+NLu9PLF8N3XN4WnEWU5bm4QLhAsNm4HqsNlC58YWHiriq9Fz4lpePvk7uBtg+o/FTBPB2rVrM202G5MmTWoCWL16dfbo0aMbFy9e3GPJkiU5M2bMqK+srLTl5uZ6p02bVvvss88WXHTRRTXr1q3LPHjwYMbkyZMbP//88zCVPXv27Oq77rprwI033ljRo0cP7+7du50ZGRl64MCB7o75lJ2fDo15UUrZgZXAKOAfwE7giNY68Ae+Dxjofz0Q2AugtXYrpaqAXsDhiDVvBG4EGDJkSHt/BEHo9Ez704e4PF4++NHMoHABgsJl6tCefFVRHwwkBZh7rM8Fc9zv3wfgwmMG8I2ThnCkwcWgntm8vvYgDpvisaW7aPb4/gHb5Payo7SWl1ftw+3x8viy3dwwbThPLN/NScMLmDO5PxceMyD4DKXA61cvSoW4VEKuh5LhsJGfk0Fdc0PYeDKCWsPFS/T1UEET3/IS7Woyeo4plXvgg9/BqLNEuBhQXV1tv+2224ZUV1fb7Xa7HjZsWNNTTz1VvG3btsO33XbbkMbGRltWVpZ36dKl237605+WXnPNNUPHjBlTZLfbefTRR/cEAn1DufTSS6s3btyYdcIJJ4wDn1Xmueee2y3ipf3oUPGitfYAxyql8oFXgPFG0/xfjX4Kjcx8jwGPAUydOlWsMoJggUaXhwy7LewgX7P3CJ/uLKem0ff797T5HwK+8zDU3rngllPRWvPE8t3c7a9ZooCe3TL49/Un8PCSndx7+eSwANMJA3oA0OT2BEWQ1pp/f7yb5z7/KjjvieW+axdM7h9lsVCEiJfQ8YB1wqJrxUgkJEq4OIlnWbFmeTGaFrjX1HqjNTw5BzxNcNKNMZ/TVZk+fXr96tWrt0SO9+/f37127dqo8ZdffnlP5Nhtt91Wji/2Msivf/3r0l//+teGTQSF5JMS2UZa6yNKqcXAyUC+Usrht74MAgLpZvuAwcA+pZQD6AFUdMR+BSHd8Xo1dc1uumc5qWpwMf1PH1Ld6KaweyaL7phBjxwnF//j46j7bjp9BHecPYafvbyOV9ccCHFjKL49fQT3vruVJndL8OyscX2YNa6P6T4iD2GP1/jfG8YuFBUUUaHrmFlezLAnWbwYbTZeTEwYKuxL+KXQax4XNFZD4xForPL9V7ETqvbCqbfBiJmW9y8I6UZHZhsVAi6/cMkGzsIXhPsRcDm+jKPrgNf8t/zP//5T//UPJd5FEFrHfe9v5R8f7eTTn5/BWfctCcaBlNU0sXR7GXnZ4Rk3d188kTmT+5OfkwEQrJsSaWUIFTNWiJxl+hNt5ELB2PJitrbJMnHdOFaIF/MSpm3iuY1CrSvuZljzLNSWQVM1P27aynO2k7ikag/8/mzjBRxZcOr3E/wEgpBedKTlpT/wlD/uxQa8qLV+Qym1CXheKXU3sBp4wj//CeAZpdQOfBaXqzpi04KQSni8mpueWckxg3rwndNHhLlmYvH6Wl9PnL8t2k5ds4feuRncddEEvvef1WEBr/k5Tu6bdwxnju8bdn8gADXyHA4evK37OOhoT7D5egq8fiNPuDgwd60YipckqBebMn7dMmbdrWQLCkB8wuWNO3wDzhzOdbsY7dhC39o6GDAFjrkKsnpAZp7va1YP6N4fuvVq4ycShNSmI7ON1gHHGYzvAk40GG8E5h2FrQlCWvDamv28vvYgizYfYtHmQ5TWNHHC8AI+31UerD9iRkAkPP/lXgb0yGLZ/53BextLouZdfeKQKOHiwzioNPjWqh6ImJeILTXM8hLbaxNyzSATKAluIxVHnISnSoe883ph9TNw5CtoqID6cm4u/ooRjkE8pq+BFf+GvhPhxsVgd/LgPT/nR00PgQc45V6YdHmb9y4I6UhKxLwIgpAYJVWN/OD5NYCvjDz4Uppv81tN4omXUH49pwi7TQUtEN0y7EE3ktmxbosjUqwGy0bOS8BrFB7zQrR6sSpJ7EmxvMSp82J2ffdieP02UDbILoCcAvJc9dziWE1f6qFkHZz/F7D73HhvO89iQ003xg/uy08nXNrmfQtCuiLiRRDSkJ1lLYU7e2Q76ZbpCDv4tdaU1zXTOzfT8P7Qw35wga9kReAA9oSYP8yMEoHxyHM/0WDZ0Hlam1teDKvN0hLgG255MVcvxjEvyXUbGVfGjZhbdxhWPQVb3wZnDvx0Fzh9hf8efX4xt27+OrNZDn0nweQrgvdqZecj73HYc/qEF48RhC6G/O0XhDSkOSSjJyfDjiK8Guhzn3/F1LsXsbWkxvKagTptoRk/ZhaUwIEfeT2y1ko8Qudp//8M55kIkRa3UbRlw2oQb1ICdkNWjhfzAgo+edBXi+XwdjjxO0HhAlCXUcCUpkeZ7ngOblnui2MJ3JmwX04w4umnn85XSh2/evXqrI7eC0BJSYn9pJNOGpOTk3PctddeG1agbNmyZTljxowpGjJkyMTrr79+sNcf6HXo0CH7qaeeOnro0KETTz311NFlZWXWAt7iMGPGjFGHDx9u01pvvPFG91mzZo1Kxn7MEPEiCGlIk7ulSmxOhgNUuMtl2XZfr51dZfFbq0RmCLm98S0vgcO4rZaXKEwtL0ZjCq+Ocd0wYDd6LBluozDLisF6gZFh6iD56x6HDS/D8NPhZ8Vw9u8MZiuMPlWbv78CAM8//3zBlClTap955pmCjt4LQE5Ojv7d73534K677toXee3WW28d+tBDDxXv2bNnw65du7IWLFiQB/Cb3/ym/8yZM2uKi4s3zJw5s+bOO+/sl4y9LFmyZEfv3r098Wd2LCJeBCHN8Hg1mw5UB99nOGy+Q83g4LdyyAWsBoF6J6Gum3i3R4mBoNXDYsxLZMCuxXmBMR0jYDdej6GWdZIb82JoefH/pn3Q+SC9l9/lcxsdd63hWrFaAMSyKgnWqKqqsq1YsSL33//+955XXnmlZ2D8ggsuGPHCCy8EzVyXXXbZsCeffDK/pqbGdv75548YM2ZM0QUXXDBi8uTJ45YuXRrVhOu1117rPn78+KIxY8YUzZs3b1hDQ4MCGDhw4KQ77rhjQFFR0fgxY8YUGVl78vLyvOeee25tVlZWWIfR4uJiZ21tre2ss86qs9lsfP3rXy9/9dVXewK88847+TfddFM5wE033VT+9ttv94xc1+12c9NNNw2aOHHi+DFjxhTde++9vcFnGZk6derYs88+e+TIkSMnfO1rXxvi8fj0ysCBAycdPHjQUV1dbZs5c+aosWPHFo0ePXrC448/3jPW51ywYEHe8OHDJxx//PFjFyxYkB/YQ3V1tW3evHnDJk6cOH78+PFFzz77bD7AihUrsiZNmjR+3LhxRWPGjClav369sY/bBIl5EYQ042+LtvHAhzvCxmxKmbpcjAidGzgQDS0QJgd70G3U1piXkGPYF/OSWOmmljovsQNmY2FPghIIDz8xspgoxqqvmGTbQ8W0uyg483bTjcZyDLXUgGnTdlODV787mNJN0Z0420Kfonou/kfMho/PPfdc/syZM6smT57clJ+f71m+fHnOtGnT6q+88sqKF154oeeVV15Z1djYqD7++OO8p556qvjPf/5zn/z8fM+2bds2ffnll1mnnHLKhMg16+vr1U033TT8vffe2zp58uSmSy65ZNi9995beOedd5YC9O7d271p06bN8+fPL5w/f37fF154odjKxykuLnb279/fFXg/dOjQ5oMHDzoBysvLHUOHDnX5x10VFRVR5/lf//rX3j169PBs2LBhc0NDgzrhhBPGXXjhhdUA69ev77Z69eoNY8aMaT799NNHP/300z2/+c1vVgbuXbhwYV6/fv1cixcv3uF/nt3sc/74xz8u+973vjfs/fff3zphwoSmOXPmBNum/+IXv+g/a9as6pdeemnP4cOH7VOnTh1/0UUXVT/44IOFt95666FbbrmlorGxUbndiXVSEMuLIKQZK4orw977+vq01DyBUOtJ/FMu5r/0490TNZ7YqWrZ8mIYBBviNjIImDXciaFVJAluozgxL0rBubYVeLWifuylMdWHNctLZ1AvHcOLL75YcPXVV1cCXHbZZRUB19Hll19e9cknn+Q1NDSoBQsW9DjxxBNrcnNz9SeffJJ79dVXVwCccMIJjWPGjKmPXHPt2rVZgwYNapo8eXITwPXXX1++fPny7oHrX/va1yoBTjzxxPq9e/datjAYiflEfsYWLVqU9+KLL/YaN25c0XHHHTe+srLSsWnTpiyASZMm1RUVFTU7HA6uuOKKimXLluWG3jtlypSGZcuW5d1yyy0D33nnndxevXp5zD7nmjVrsgYNGtQ0adKkpoCFKLDO4sWL8+6///7+48aNK5o2bdrYpqYmtWPHjoxTTjml7r777uv/y1/+st/27dszcnNzE/qXi1heBCHNMLKQKMItL8GGYGbZQqGWigi3Udg805gX/9eIvVjuehx8djgm3QGMM4eI09soTr2VAMnONopa79Amxn7x/5jieIs1eiSF3QpjrtUivuILnLQmjoWkPSgpKbF/9tlnedu2bcv+3ve+h8fjUUop/fDDD+/LycnRJ598cs3ChQvzXnjhhZ4BwWLFGhhvTlZWlgZwOBza7XZb/tMbNmyYK2BpASguLs7o16+fC6BXr17u4uJi59ChQ13FxcXOgoKCKNOF1lrdd999X1122WXVoeNvvPFG96heYRHvJ0+e3LRq1apNL7/8co9f/vKXAxctWlR96aWXHjHbq5mo0lqzYMGCHcccc0xT6PiUKVMap0+fXvfKK6/0mD179piHHnpoz0UXXWQ5w0AsL4KQZpiVnw/9/ZlQsTcTIQLmB6hZVpEyeGWVWG4j41iVkM+pwsfN74kebfcidcv+Qt/dr7JVD+I3ruvjtweIaXnpRG6jDuCZZ57peemll5YfOHBg/f79+9eXlJSsGzRoUPN7772XC3DVVVdVPPnkk72//PLL7pdeemk1wKmnnlr7/PPP9wRYuXJl1rZt27Ij1z322GMb9+/fn7Fhw4ZMgKeffrrX9OnTraf6mTB06FBXt27dvB988EE3r9fLc88912vu3LlHAM4999wjjz76aC+ARx99tNd5550XJSzOPvvsqocffriwqalJAaxbty6zurraBj630ZYtWzI8Hg8LFiwoiNzvnj17nN27d/feeuutFbfffvuhNWvW5Jh9zmOPPbZx3759GRs3bswEX0B0YJ1Zs2ZV33fffX0DWVIff/xxNsCmTZsyxo8f3/SrX/2q9JxzzjmyZs2aqO9rLMTyIghphll2jNGxn8gZF68nj9F4dIXdBA/XiInmAbvGezOKeTFZ2j8vGnui/4Rb+STsW9HSDLGhgmFVJXye6eLPrqvI31kGe+uh/rAvMHfbu5QNm8u8Tb6icvG7TmM6ryUeRtRLa3jppZd6/fSnPz0YOjZ37tzKZ555puC8886rveSSS6pvvvnm4WedddaRgLXkJz/5SdkVV1wxbMyYMUUTJ06sHzt2bEPPnj3DsnFycnL0I488smfevHkjPR4PxxxzTP2Pf/zjskT2NnDgwEm1tbV2l8ul3n333fy33npr2/HHH9/40EMPFd9www3DGxsb1axZs6rnzZtXBfDb3/724CWXXDJy6NChvQcMGND86quv7oxc84477ji8Z8+ezEmTJo3XWquCggLXW2+9tRPg2GOPrf3Rj340aMuWLdknnXRSzTXXXBMmflauXJn985//fJDNZsPhcOiHHnqo2OxzZmdn6wcffLB4zpw5owoKCtwnnXRS7ebNm7MB5s+ff+DGG28cMm7cuCKttRo0aFDTRx99tOOZZ54peOmll3o5HA5dWFjouueeew5E7j8WIl4EIc2IEgwQVm02UVoOS/Nr0eNxAnYTfDb4g4gTag+gDNsDJLqLhOJ0qg/A67dDdj7k9vX3EhpAQ8EkXBvf476MR3ytZYFg1dz8oRwafSVschnsNbH9SJmXtvHFF19sjRz71a9+VRp4nZmZqY8cObIm9HpOTo534cKFu3NycvTGjRszzznnnDGjR49ujlxn7ty5NXPnzt0UOb5///71gdenn356vdEeIueFcvrpp9dv3759Y+R4v379PJ9++uk2o3sC2O12/v73v+8H9kdey87O9r755pu7zPZx2WWXVV922WVRn8fsc15++eXVl19+edQ+c3Nz9X/+85+oAOV77rmn5J577onuSWIRES+CkGYYWQp8Z1lYjd2Ya1jNNooX8xLtN499X7z1E2nM6LO8RF+P9Wyja0axPgCUrIcvHoOGSmg44vtaUwJouGER9G6pwVVRXseFq95krNrL/GtmMXLYMMjKD6Yh1e48DHwOWHdTSap0alBTU2ObPn36WJfLpbTW3H///cUBq4zQcYh4EYQ0I1JkdMt00ORuNrS8WLMq+OYYu43MYl5C7zRasXXHq2l7ABOrkDaqsBt6j9Y+0XGkGKoPcG7zZzSo4ezThZxi28QW7xAcuhmq9kFtKdSVtXxd8W+oL4f8IZDdE/KHQv9jod+kMOECvu9dNd34Uo/DXTAacrpHXY/cnxlmaei+ewPXRL4cLXr27OndsGHD5o7eRzKZM2dOzZw5c9ock9ORiHgRhDTDHlJU5LYzRvH1k4dyw1NfRvQ28n21JF1M4ldi0VJh1zhktzV1XiAx8YJZqrT/9YnedTD/m9DUkmjxI+BHmbDT25+RNn/owxsmm8sugK+9AMOnx/8cYdlGBtfD5saJeQkKQ3NLWBpLF6/X61U2m00sF0JMvF6vArxm10W8CEKaEVpU7YfnjAX8RepCTv7EUqV9GPX5M7/f+Hqih2vo/VrHchsZB68G+jAZNUa83PsO2Oxw3nzoMRjyBvDUM09wXeOzQeHysmc6EyYex7hRI6FbIXTrA7mFvtcZ3Sx+ivhdpUMzueLGvMSYF+tamrChrKysqLCwsEoEjGCG1+tVZWVlPYANZnNEvAhCJ0ARo0aKRQzrvLQyVdqqWyM8YDdBt5FhkK6PG+xvMUt/DmO/ASffEhz/b2YNC6rGcbX9Az7zFvE/72k8VDSFcZP6W9qvGeGWH/MsIbCQbRRLAJp839MFt9v97ZKSkn+WlJRMREp1COZ4gQ1ut/vbZhNEvAhCmuEyUilKRbiNEgnY9buAEgjYNSsEl6hFwGqFXcN7TdbJ0I3c7HidSvLoeead4ffYbKzXI1jvDlYvT0pX6fDeRrFjh+I9riXd3FwEpWvMy/HHH18KXNTR+xDSH1G+gpAmLNlWxr+W76bZHe0GDg1eDRu3cMYF3UYWK9KazfXNb9uhmlhxPWNBcEnxHyhUVTxpvxy6943YXzTJKVIXul6863EsLxFfjdZJT+kiCMlDLC+CkCZc968vAJgxJrq8fJssGP57E2kPYJplpMK/xn92aNBL8P9izzPYQ/B6cz1jqz5muWcCCzPO4w6T/YWSnPYAsS0vYWNxTS/m81TUC0HomojlRRDSDJfHzPISPdeKJSRWp2Kz+82aGbacu4mfrhptHvNiNGakB3Z9hFM38bDnIrw2Z8x7AphVLE4Eq9YU39zYa8VqLNkSayTqRejaiHgRhDQj4DbKdtqDY0pFNGaMY3oJyzYKWF4SiHkxXTfB9gDR2Ubx5wXHDD4Dm/5Hg707n3vHW9uAydqJEiZODL6P8SwzRvuJHfOS6A4FoXMh4kUQ0oxmj5djBvXgs1+cGRyLtLx4Eyn04qc1dV7MaM3Z6tU6RmPGWD4U//WSDbD+JTYUnIUbh2VLUrItL/FiXiynShtdk5gXQQBEvAhC2tHQ7KFPXhY9slvcIkoZixezQ86onopxnRcT91DcLCSTB0fOD9mhpo3ZRmv/CzY7Hw242XQPCbUHSGQvId+7eA0urVteDK6JbBEEQMSLIKQd20tryYhocKRIzG0Udq//PEws2yjemtYO2Si3UWvrvGgNW96AETNpcuaZP89w7eRaXgyfkYDoCMw1FDkJikNB6KyIeBGENKS8rinsvanlxcIpF5iTWLZRbIuM1bM1TH8QI+Ylzh6yjmyFyj0w7oLYQcMGHygZbiPDzKcQbHEsM2FrxbjcloBoQRK9cFoAACAASURBVOhMiHgRhDSkuLw+7H2keEnI8uL/mkjMS1xR04qAXWLFvMTRIXl73vM9dMzshF1XyS9SF/u69fYA5mJSLC9CV0fEiyCkIQerGsPeJ8VtZBTzYnpPYrEwVoi95dhWoe573oVBJ0D3vqatC8zGzNK+EyFeTEtYNlJc9RJr/4llcwlCZ0XEiyCkIT89b2zY+0QDdq1m3ZiJFLPzPlG3RljAbqIxL/57+1NO9uH1MO6CiHusucGORpG6RNoDxNJSLcuIehG6NiJeBCFNCATp2m2KW2eOCrumVLjlIp7hJay3UYwA0cQr7CZmGQid59XatKt0rHvPsq/0vfCLlxYBZXCPwVhSso3Cgl5iX4/vNjKfIG4jQfAh4kUQ0oQMh+/H1WFkIUGFxYu0BOzGX7dV2UZxK+wmTkzLS4z7zrGtoDl/FPQeHXeyVWtMoiQW89KWgF1zl5IgdCVEvAhCmhBTvERYXgKNp621B/BhqEfMYlviLNYaQaCJ5TYydsXkUcfJts3UDT83ZAuJBQ0np0hd6GvrItCIWHPF8iIIPkS8CEKakOlocRsZEZ5tlHi6kWHMi9ktcUSN5ZiXkHV0DLeRmQtopm0NTuWhdsR5IWvGvieS5HSVjh3zkoxMrrA5YnsRujgiXgQhTQhYXpz26B9bX2+jFrwJiJeWDJYEYl5MxxOMeYl4n0jAbo6u51rH+xzS+TT3PTZqTaufx+Db2SasVvY1vz9WzItkGwkCiHgRhLQhNGA3Epsi7OQPvEwkANZobdOu0nEtL9aIqrBrYR4AW9/hr/uuYIrazn3ueShlIOgM92c9o6q1tFW8xFw74qsgdFVEvAhCmmEcsNsS50LIaysGmPAaJebXIp8Xcy2r7QFCXntjqZdQtIb3fkWDLZdLmn/Li55Z4anIseJCjCwvSRYv8VKl4xEzYDf42US+CF2bDhMvSqnBSqmPlFKblVIblVI/8I8XKKXeV0pt93/t6R9XSqkHlFI7lFLrlFJTOmrvgtAR5Of4GjF+a9rwqGs+t1Go5cX32ooWCD0GIw9e02yjdjg8fe0BLHSVPrAayrezsOe3WKtH+a+bzLVAsj+LccyL9ftjpkq3ZkOC0AnpSMuLG/iR1no8cDLwXaVUEfAz4AOt9WjgA/97gNnAaP9/NwIPH/0tC0LHkZ3h4JjB+Xx7+oioaxFeoxa3kQXTS6xgU9Nz3dQik1gqb1TArpXmRpteA2Vnbc4phvtsCdi1FoBsVFm4LRgJlYQaM1pI9RbDi9DV6TDxorU+qLVe5X9dA2wGBgJzgaf8054CLva/ngs8rX18BuQrpfof5W0LQoehtY7prjGqsGuF0DXbGvMSuh9Lzw6NecFCY8ZN/4OP/wpjZ1Pv6GG4z1ieq/aqsBv+jLZaXuJfk2wjoauTEjEvSqlhwHHA50BfrfVB8AkcoI9/2kBgb8ht+/xjkWvdqJRaoZRaUVZW1p7bFoSjjvk5a5xtZMltFCPmxdzCEmdNy+0BII86nnX+gRdrr+fUhsX0o5wT1JaIPSrweuGdn4EjC874dUQnZ7M38feVjDovcUko2yj+NbG8CF2dDhcvSqlc4GXgdq11daypBmNRv5u11o9pradqracWFhYma5uC0OFoHSPLR4W7iIIvE0s2iqqcaxrzEuc3h9XDdWDZUj7I/DEn2bbgxsF3q//KZ1nf56XM3zGQMgIfQAFsfxeq98Pcf0CfcfHL7FtMlT4aQiChOi8xmy8m5pYThM5Kh4oXpZQTn3B5Tmu90D98KOAO8n8t9Y/vAwaH3D4IOHC09ioIHY03ltvIYC5YS5UOtUZEZt6YF6OL4zaK+1QfRbufolBV8R3XD/l51i+x4WWzdwgAH2f9gK2Z13Gc2u5TYx/eDQUjYfxFUXsIi3mx+OwAyc42MiKRJwS2Y+T5E8uLIPjoyGwjBTwBbNZa/7+QS/8DrvO/vg54LWT8Wn/W0clAVcC9JAhdAa1jF40LC9gNuScuYcGu1iwvLQesSXaQlcO1rpw+lat4wH0xi73HsVMN4+reC5jdPJ//uM/gFc9puHBwp/MZxn9yOxzaAKf9ABwZUc8I7x1k/sijEfNiRGIVduNnG0mqtNDVcXTgs08DrgHWK6XW+Md+AcwHXlRK3QB8BczzX3sLOB/YAdQD3zy62xWEjkWjTS0eivBU6aDlJcE6L5HVZuNV0jVf08Lhuv1dbHh53zMV8H0+N7508F+4vw3AbxxPca39PdyltTBuDky63HBv4ZaXxGJezJpMJpOEKuxaWEeki9DV6TDxorVejvnP4JkG8zXw3XbdlCCkMFpj+hNjs0VkG3n991hYN3RJq6nSSTnvt7xJfWYf1jcOB4wr7P7WfR2/d1/DM9ecwmmjeofvLSzDqC2Wl4R3njBJK1KXYNNJQeisdHjAriAI1tCYH7QKFZYeHSxS19Y6LzEsPbHXjPPQ5nrY8QH7+s4icBJrbWwp8mIzfpqp5SUxjka2UfItL6JehK6NiBdBSBN8dV7Mc5dDz/1EkozCLC8W3UamIsrqmbr0z+BuYF+/s8OGE+nFpExexxozIq1iXiRgVxAAES+CkDbEDNiFMMWSSJ2XUKxm3phNC1hOYloGGo7AZw/D5Csp7XVicNhrtcJucA/G5hYrFWpDORripTXZRsbrSKq0IICIF0FIebTW7D5chyZ2AG14kbrAvfHXN8vaCaxr9jyra0bx+SPgboSTbkaFmHDM3EZgLIaUyfXA3oyWMtrW0Yh5SazOS/yLYnkRujoiXgQhxXli+W5m/WUxmw5Ux4hBiSxSZ71KXdiaKvKa2T3x1jThwGpY8meYNA8GhvdW1TGcRvGKy7WmHYHdphjRu1vKxbzErhAc+CrqRejaiHgRhBTnvY2HAGhweWLXeQl531rLS+Ty5jEvrbS8vH8n5PSC8//if16k5UUbiol4MS1hry2e6xcfO5APfzzzqNRMSV7Arkp4PUHojIh4EYQUp7SmEQCnXcWoeBvZVTp2zIvVvo2mlp44h6fhfV99DruX+grNZecHJrbsyb8vQ6FiWOo/2lUUvgejfYXeb7j1diFZXaVb1hOEro2IF0FIcUprmgDweGN1lQ4vUpdIhd1w94vVOi+tsLx8+nfILoCp32yZF3LZV+dFW44PMbW8xCpSFyp4LD0lOSTWVTq+20hML0JXR8SLIKQ49c0ewOcKstoewOO13tsobJ0470OflxCN1bD9PV+F3IxuIeuEyRfTjCrD55nEvMTO1gl92tEjaUXqAgG7bdyPIKQ7Il4EIY0wD6BVEW6jOOuExbkow3Gj9y3j8SwvEde3vu3LMJp4mek9Xh0oxGfVBaRMXvswzDbqoFM/McuLhTmiXoQujogXQUgjzFOXw7ONWt/bKPJXgvHz4h3GUZc3LoS8gTDoRNN5WmtfIT6LlpewsVZkHh1NWmN5sdQmQBC6KCJeBCGNMG8PEFFhN5BtZGHN0CUdEQ8wL4qXQCBvQyXs+AAmXBJVwjd0vsbc8mIkokJHQrcd+2BP/UM/WKcmxh9eKgo0QTiaiHgRhBRk8dZS3l5/0OBKLMtLy/sWy4vxCRg6HNbbKFK8mOwvruUl9PqWN8HrggmXxpyn/eqlNZYXy9lGaXDoW3IbtfsuBCG16bCu0oIgmHP9v78EYM/8C8LGY1lCjLKNrBDb8hLD1GNxTTYshPyhUUXpItFax7C8GD3DOHPIasBuqmLFxZQOIkwQ2hOxvAhCGhEr+8ewzkuCKTVR4sVknpnACPY2ClyvK4ddi30uI0PrSMuYr86LNrTqxLOitKbCbqqe/+nt9BKEo4OIF0FIIxKusGtigzE7+B12qzEvFve5+X+gPaZZRpFuI1//JgORYzgWuh8VNfdopkInE0tF6sT0InRxRLwIQhph7lJRhjEvVgiLebFapC5O0Evw6saF0GsU9JsUdx/a31XaquWFMMESb27gWuof+qJLBCE+Il4EIY2IZXkJtTUEs41MNIzZeLTbKJ5sMEEpqC2FPct9gboWulP74nV1XCtLrLFY45HXUtUyY0VgicARujoiXgQhjTA72GzKWJAkGvMSVefFVCBYsLxseg20FyZGZxkZ4WvMaGZ5iZ1NZChujPaVBoe+td5GafBBBKEdEfEiCOlEjGwjI1dRotYFqwG7cRszKnxZRoXjoc9483khrzUBt5HFbCOzmJdY93SSQz8dRJggtCciXgQhjYiZbWQwblbnxQx7VMCumaUn9umZ03AIvvo0rtUlqs4L1i0mZm0N0j2Y1VKq9FHYhyCkMiJeBCGNMBMNChO3UYLrW7a8mI37Lwwpec/3dIPCdOHrhMS8aPztAVpjeYkeN/zsaXDqWypSlwafQxDaExEvgpBGxGqUaGhlMVEvZuvYLbYHiGd5GVryLvSbDL1HxZwXivYndke1VzLZR7hgseo2Sn0k5kUQ4iPiRRDSiFhHlqHbyES9tDnbKMZGBqkyeh9ZZylQN6rOi0nMi2HArplgibG5dHApSbaRIMRHxIsgpBGxukonI/c3MtsoXjqy0SO/YV+EF5tpYbqwdUJeB1KlDcWLlQDhGGsbjaXq+W+lq7QgdHVEvAhCGmEeg6JMAnYTW99uLVPaPGVbe5hnX8z+fmdC/pC4zwu3vPiyjaye2WZBuul+5geMX7G7Sqf7pxSEtiGNGQUhxfB6Y51aJsPKOLMo8YBda+rFKC4FrZlf+0t6qRq2DLyAwZaeGFmkznpBuoCFJvKa1SJ1yebb04ZT7/IkYSXJNhKEeIh4EYQUY8Qv3jK9ZhYoazNNlU7s2ZEBu2YYWl72r2KyZyOlOp+SPtMSezCtiHmJ+Bo51+ijR7qpksmv5hQlZR1rvY2S8ihBSFvEbSQIaYR5irJZkbr4jRlDiWrMGKOibxRr/0sTGZzZ9Be0I9tkp/H2YT3mpSU2xFqGlNHcVMRSqnS770IQUhsRL4KQQjTGcTvE6vJspT1AwLVkZpGxWxQCUePuZtjwMp86T6KGHOtxKxHvvTqRUv/K8Fq6H+yWitSlgQgThPZExIsgpBCHa5tiXjdNozVzG4W8/mJ3BcN//hYriytM17feHiDiyp6l0FDBR84Z/uumj4i5jjeRInXBNWJfTzekSJ0gxMeyeFFKTVNKfdP/ulApNbz9tiUIXZPSmjjixfSgjh/0smx7GQAf7yg3XT86VdokNTtyYNP/ICOXVc7jTNe2so7Xq40bM8bIe44UdOl+sFsrUicIXRtL4kUp9Rvg/4Cf+4ecwLPttSlB6KqUxRUv5nVezOJbEiEq5sVKhV2vB7a8CaPPwaUyYt4XD/PGjDGCeCOzjdL8aLf0vUt3hSYIbcSq5eUS4CKgDkBrfQDo3l6bEoSuSly3UaIxLwk+P6o9gJV97F4K9Yeh6KKQ+yy6fiKmebR1y4sy1i7BgWRnEx0tLFXYPQr7EIRUxqp4ada+SD8NoJTq1n5bEoSuS6PLG/Y+snZLLDGRjFTpgpyMqHWNCLOOrHwSsnvCmNlx74skcp5XG0fsxrLFRNV5sfbo1BUAkiotCHGxKl5eVEo9CuQrpb4DLAIeb+vDlVL/UkqVKqU2hIwVKKXeV0pt93/t6R9XSqkHlFI7lFLrlFJT2vp8QUg1mt2R4iX8eqyYF8MidQmql8uOH8QfL5kUtnIs8nSNz2V0zNXgzEroWb7VIwN2jdOwjdxlLZaXSFdXep/s1lKl0/szCkJbsSRetNZ/ARYALwNjgTu11g8m4flPAudFjP0M+EBrPRr4wP8eYDYw2v/fjcDDSXi+IKQUUeIl4nqsRonxso2sYLcpvnZSS1l/U8uLX2Gc5V0OXpdPvITtp3WHq9bGdV6MCHwvWpttlKpuJWup0kdhI4KQwsStsKuUsgPvaq3PAt5P5sO11kuVUsMihucCM/2vnwIW4wsWngs87XdffaaUyldK9ddaH0zmngShI2n2hNd5iSw8Z1YAVyllqc5LosQ7I8/3LIa+E6H/5ITuM5toankxutU/GCl20v1clyJ1ghCfuJYXrbUHqFdK9TgK+wHoGxAk/q99/OMDgb0h8/b5x8JQSt2olFqhlFpRVlbW7psVhGQS321kMXU5cH8b9xPLCjBS7WeC3g7HXNXyPB24z+L6Ee/N6rzEKlwXVaQuzU92aQ8gCPGx2tuoEVivlHoff8YRgNb6tnbZlTFGP65Rv5u11o8BjwFMnTo1VS3DgmCIyxP+V9Zq+rPZYZZozEvUujGuXWZfhhsbjklXGNxnNdsofJ5ZV2nD9UzSjQLDqfjDP75/Xtw5VtxmEvMidHWsipc3/f8dDQ4F3EFKqf5AqX98H4Q1qh0EHDhKexKEo0JTGwJ22wPTc9TdwCX25XxuO5bTuve1fl/kPIMxy72NTNZI1YN95x/PT55LKDU/oiAcNawG7D4F/BdY6f/vP/6x9uB/wHX+19cBr4WMX+vPOjoZqJJ4F6GzEek2iiRWwG57YPa8Hh/fQ18qedE+J+n7sFnMgbTcdylFsNtUMNA5JlJhVxDiYsnyopSaiS94dg++n5vBSqnrtNZL2/JwpdR/8QXn9lZK7QN+A8zHl5p9A/AVMM8//S3gfGAHUA98sy3PFoRUpNnTWsuLMW0O2DVa2OMme8sC3vKexBe2YxPaj5X1E802Mr+enlgqUpeqCk0QjhJW3Ub3AedorbcCKKXG4LPEHN+Wh2utrza5dKbBXA18ty3PE4RUp9kdnm0UGfNinm1kPJ6MlgFR7FmGvaGC1z2nmE6x7jYychFZdBt10vNbehsJQnysFqlzBoQLgNZ6G77+RoIgJJFIt5HXaraRyXi7WF42vYbXmcNi7zGx7mz1+sZp0daq7prNTScsxcWk90cUhDZj1fKyQin1BPCM//3X8cW+CIKQRKLdRtbaA5jR5lTpqEIsHtj8Ok3Dz6ZpXYbxTbTtcE20zovZ3HifPVXPfylSJwjxsWp5uQXYCNwG/ADYBNzcXpsShK5KvAq7Zieueap0YuNx1y3+BOoP4xrra8I4pCDH+D5ry7ct2yhGh20rpGIqNYgwEQQrWLW8OIC/aa3/HwSr7ma2264EoYsSt0idiSwwC3Jta8xL1LKbXgNHNnmTZvN4di0nDOtpcl/rc6WNMnISSX9O1VRpq6T37gXh6GDV8vIBkB3yPhtfc0ZBEJJIZJ2XSO1hGrCb4HNaFVDr9cLm12H02ZDRjbOL+pKfY+w6sm55iZ5pJZsY4n+GdBUBYnkRhPhYFS9ZWuvawBv/a2N7sSAIrSYq5iVCvSRa2ySpAbv7voDaEiiam9h9Cc4zzkCyNi+RZ6cu8T9AW/9cBSHdsSpe6pRSUwJvlFJTgYb22ZIgdF1c8eq8mB3Y7VVhN/TNptfAngmjz2mXZwUwKlLXmoDddCWW+Er3TCpBSBZWY15uB15SSh3AZ8geAFzZbrsShC5KdKp0Wy0vxv9ETzhg1+v1iZdRZ0JW/P48bWkPYHhAW8xACn12uhonRJ4IQnxiWl6UUicopfpprb8ExgEvAG7gHWD3UdifIHQp4mUbJZwq3eYT3P/EA6uger8ll5HvLqt1XoxiXqwF7Ab7MkbNT+/jX6wrghCfeG6jR4Fm/+tTgF8A/wAq8XduFgQhecTNNkq0SF0b9xNcdtOrYHPCmPMs3pjg+nFu7UoxL1YDlgWhKxPPbWTXWlf4X18JPKa1fhl4WSm1pn23Jghdj1YH7JqsZ2R50TpBt47WPpfRiJmQnW/9vlbS5bON0nbngnD0iGd5sSulAgLnTODDkGtW42UEQbCA16txeSLUhtWA3XbqbaSUgoNr4chXll1GwfuszDMYM3YbWSfdj/50txwJwtEgngD5L7BEKXUYX3bRMgCl1Cigqp33JghdikirCxjEvJgcbFY7MSeKAp/VRdlh3AWJ3WdlnsXKuVbHYo0LgtB5iCletNZ/UEp9APQH3tMtqQs24PvtvTlB6EoYiZeobCOTe9utPQDaF+8y/HTIKbB2U4z9GD0hEsu9jeKsmK7ZRoIgxCeu60dr/ZnB2Lb22Y4gdF0ig3Uhuqu0Uel8iBHz0sY9ZZRvgYpdcOptCd1nPdsouWOxxsE8dVwQhPTCapE6QRDaGUPxEqlezGinErs5O98AZYNxcxK6r21dpS2mSrfDswVBSA9EvAhCihAQL057y+kbnSptfG97WV5ytr8BQ0+D3MI2rmSM5SJ1rZjXlrUFQUhtRLwIQooQiHnJdNiDY56omJcEs41M1IuVM3yU2oezcntCWUbB9a3OMyxSZ21BcyEnAkUQOjsiXgQhRQhYXjIcLT+WltsDmBzYbUmVnm37Ao1K2GXk31Crpxm6jRJoDyDaRRA6PyJeBCFFCFheMuwtP5aRAabJivOwEgpzvv0LmgecCHn9E1uctlk/LAsVs1Rp/1ejjygBu4LQORDxIggpQsDykuls+bGMzJ42q+eSSIVdKwxXBxlv+4rG0dZru4Ttpw1Vco0tLxbdSyZzBUHoXIh4EYQUIeg2soeKlzZ2lW7NRjxu/uB4AoCmUa0UL5bnWXMRWb033rNF2AhC50DEiyCkCFZiXswwbczYGvWyewmn2jfxkPsidI9BrVggkYyh6DGr7QE6a28jQRDiI+JFEFKElmyj0JiX8DmmJfFN1jQL2I158G98hRqdzd/cl7ZaALStzou19aTOiyB0XUS8CEKKYCnbyOReUwtLou0BPC7Y8gaLvFNoIqPV5ou26AfDPkYJuJdiBQtLwK4gdA5EvAhCitAiXlrqvESKF7MgVTP3UsJH9drnoaGSNzwnA63PGmpLwK7lsTg1b0SmCELnRcSLIKQITQap0tF1XowP7MhidgESsjS4GuHD38Pgk/nQe5z/edZvbw1GAsRyh+wEqw2DBOwKQmdBxIsgpAjBVGmHeaq02dlruQdSLDa+ArWHYNbP0f5fDW2o1mJtlmHAbnKeIDJFEDovIl4EIUVoS8xLZEp1gIRCPDa9Cj0Gw/AZLc9rpaWiLQYOq/EtpnsT1SIInR4RL4JwlNmwv4r1+6qixl0G2UZRsSymbiPjZxkNG8bHNFbDzg9h/IVhz2h1tpHVeRYtLwl1lRb1IgidHkdHb0AQuhpzHlwOwJ754QXgmt1ebArsIae3N6rCrvGaZm6jUJ0SeG04c/t74GmG8RfF2rplLNd5MbSyWLW8mD3b0qMFQUhjxPIiCClCs8eL0x7+IxntNjI+mc2zjXTUHMMg3k2vQW5fGHxS+PPaOVW6PYrUxeptJAhC50DEiyCkCC6PNyzTCKx3lTbPNgp5bTAGQHM97Fjk6x5tC39+u6dKG4wZF6mzVvvFbK4gCJ0LES+CkCK4PRqHXYUd/FEhLyb3Wsk2CgihKCvNjkXgqociA5dRqy0vye0q3Zp5ImEEofOSduJFKXWeUmqrUmqHUupnHb0fQUgWbq8XR4TlxWpjxsiU6gDawPQSpXM2vw7ZBTB0WtT9rXYbtalInfXg3LY8e3jvbgmsKghCKpFW4kUpZQf+AcwGioCrlVJFHbsrQUgOLo/GGeEziXQHmVk0zNxGoQRjXkKiQTJwwbZ3YNz5YI+O329/64W1InWxRE7kNSt7nnf8IG6eMdLKBo86Iwq7kZfl4CfnjY26Ju0NBMFHumUbnQjs0FrvAlBKPQ/MBTZ16K4EIQm4PT7LS6hAiTysEi1SFzoazDYKGTzVtgGaqmH8XMP72zt+xHKqdALWGCtbPmVkr7CsrlQiJ8PBurvO7ehtCEJKk1aWF2AgsDfk/T7/mCCkPS6vL+YllIAmCZyzibcHiF4rVBDNtn0JmXkwYgZGdExX6dZX5/VfASTbSBA6M+kmXox+XYX9jlJK3aiUWqGUWlFWVnaUtiUIbcft8eK0GWcbBVKoEw3YDXURBV4HptrxcLZ9BYw5DxyZhve3PlXaap2X1j8z8IzuWeEG5K6QbNQVPqMgxCLdxMs+YHDI+0HAgdAJWuvHtNZTtdZTCwsLj+rmBKEtGGUbBQJ2Ay0DTN1GVlKlI9xGJ9k2U6BqjbOM/LR3tVrjgnSJWV4KupkIr1bvShCEVCfdxMuXwGil1HClVAZwFfC/Dt6TICQFl1fjiIjDCAiNQP0Xs3hN02yjsLVaUqUzdBO/dDxHvc6EkWea7qn9u0pHk2goSkGOM+6agiB0LtJKvGit3cD3gHeBzcCLWuuNHbsrQUgOgYDdUCItL+7IfgF+rFheQj1LX/O8xgRbMYu8UyAjpw27bhtWK+waUdvoBqItL12hSJ0kHQldnXTLNkJr/RbwVkfvQxCSjdvjs7yEHr0BURIQLy6TDoy5mdE/ytk0Yvc2t7x3V/BHx+NM2mWjv3sVB3QBP3d9m1jdjNq7zosRVi0v5XW+z9YrNyP82a1/tCAIaULaiRdBSDfcfp9OpFUlEpfXS7eM8B/JyIBdt4l/6HtnjOKV1fvZf6QBgNNta3k640+wAdieB916c3NtA93tpRxpGEi9yuLu5quoIzvmnto95sVofYvKp6KuCYCeORHixX97ZzROdAWrkiBYIa3cRoKQjpz1/5ZQdOe7pte11ny843AwYDcUb0TMi5nlJctp58bTRwTfz7MvoULn8l6fG+DYr8GA46jIGMBv3dfy13H/4erMh3jPe0Lcvbd7zIvFOi9GnDKyFwDTR/cOX1NsL4LQ6RHLiyC0M3vK62Nef2PdQb7/39UAnDW+b9i/roOWF7/bqNksMhefECigmvnOxznHvpIn3eewuc/1nDN7MgD/fHkdzx/ey9ewXqm1tTKgLTEZdouK6Yxxfdl29+ygSy0SkTCC0HkRy4sgdDB7K1vEjdPU8uIbd5tYXsDnUvil8znOsa/kNc+p3Ou+MrzOS0iROqvaoiPcFFYDdgFD4SKeFUHo/IjlRRBSiMi4mEDxuWDMi0m2EUBewz7Os33KC+6Z/J/7RiAy20gHx6xaRtpbBxgKDREfgiDEQSwvgpBCRDVm9IuXzDjZRlQf5KzPbwDgec+s4HBYnRf/QYSdMwAAIABJREFUV7O0aiPaP+Yl+gFW3UZdkW+dNpxsp51pEXE+gtDVEMuLIHQQWuuowzs6YDcyVdrA8qI1PHsZmc0VXNH8K1br0WGXItfSOrxtQCzavTGjwVjk96C1dMZso0mDerD59+d19DYEocMRy4sgtCOeiJ5DoT2IAtdCBUak2yhwLWaqdMl6KN3IuvE/ZpUeE35/6BHuf+lNwG3U3hhpI4dNfi0JghAb+S0hCO1IXbM77H1o92ejTtBOW3hvoyjLi1EDxo2vgLKzd8C5MfcStLxYtrt0DPZE+wOYIM4nQei8iNtIENqRuqYI8RIiPoxibx12W1hMSkDgXD5lEDal+NHZ4ZYVtIZNr8Lw6biyeuHrXRp6PfplIgG77Y1RTZbI/k6CIAiRiOVFENqRWOLFyPISGe8RmJKVYecv846hV25EB+WSdVCxCyZcYljcLfQJgUf7arykhnoxchvZkxTzIghC50XEiyC0I7VNnrD3T39aHHztMcgcctpsYdaIQIyM6XG+8VVQdhh3oWF9lNBidC1dpdvf8mI1ztdomlNiXgRBiIP8lhCEdmJHaQ1XPPJp8H11o4s/vbMl+N7M8pKX7YiaY1i4TWtfvMvw06FbL0PBoCOmR46ZccKwnhZmmdOmCrtJchulhm1JEIT2QGJeBKGdeG/ToWA5/0yHjY+3Hw67HpmJBL6som9PH86Xeyr4eEd5i+XF6DwvWQeVu2Ha7f45RpaXkNcELC/xA3af/OaJlNU0xZmVBAw+V2SVYUEQhEjE8iII7YTL7ZMIZ47rA8DWQzVh143Ei8OmyHTY+fY0X5PFwBTDZoP+LCPGXQgYNzQMi3nxtgzG623ULdPBsN7dYs5JBkafS7KNBEGIh4gXQWgnqhtddMuwM6Zfd7xac6g63JJh7Dby/0j6T95A5lGUUUVrX7yL32UExvVRwmJeErC8HC2kzosgCK1BfksIac+O0lr2VsTu3NwRVDe46JHtxGFTeLyasprGsOveYJG6FikRcJkEznSjsi5Ai8towiXBoTPH9+Fbpw033U9gLa/WKZQqHU2yLC+CIHReRLwIac9PF6zl7jc3dfQ2oqhqcJGX7cSmFF5NtOXFrybcIQolYHUIxK+YWl6CLqM5wSGn3cadFxaF1UkxDNi14DbqSJLVHkAQhM6LBOwKaU9Vg4ucjNT7q1zd6CIvyxm0JJRUh1teAqLFHZIyHaikG9AfLanSIQd6wGU0YkbQZRRKmNAJDdgNSZVOFQwbM4rlRRCEOIjlRUh7Gl3eYFZPKlHd4CYv2xE8jCOzdwJWlVDLS6B7dECsBFOlQ39SD671uYyKLo67h9Dolujol47H0G0kXaUFQYiDiBch7Wl0eYwbFnYw1Y0+t5GZJSHoNgrZe8DyEji/A96dMMvLJn9huvEXGq4bOteoq7S3HQvs5mTYgQixFQPDCrtieREEIQ6pZ2sXhARpdHlwGVSr7Ui01hyp97uNTCwJRjEvLZYXH1ExL4HCdCNmQE6BhX1Ev9btmG30j69PYcGKfYzt293S/PZMlRYEofMilhchrdFa0+j24koxy8tnuyqobXJzzOAe2OJZXrxGlhcVNie4wp5lULkHJl9p/vDQ8JgQmRJqeWmvgN3+PbL5/pmjDWNZrCKNGQVBiIeIFyGtcXk0Hq+mrtkd1QSxI9haUsPOsloWbyslw25j9sT+mCXPBOJZPGGWF5/bJXD2B4vUBdZY+SRk9YCiuQnvLbQ9QMrYqQy+N2ZiTxAEIYCIFyFtWbKtjFJ/7ZS9FQ1M+M27HbwjOPevSznzviUcrmmmsHsmWU47dnvLj1nALQQtmURNbm/U9RYvUUBmKKgrh82vwzFXgzPb0n6M2gPoVKrzYlikTsSLIAixkZgXIS0pqWrkun99wTGD88PGG10espz2DtpVC+V1TfTKzQDCs2cKumVwsMonuNxezSur97Fw1f7g9UxHjDova/8LnmaYcl3MZweeZlPG7QFSRbiAcbZRe1peUrm+jSAI1hHLi5CWHKhqAKC4vC5sfFtE/6CO4lB1E726+cVLyE9Zfk5G8LXXq3ll9YGw+yLrvARCeWxeN3z+KAw6EfoWWdqD3aZiNGZM3UNcLC+CIMRDxEsCbDtUkxJxFQKU+qvVds8KNx5uLUkN8bL5YDUF3TIBsIfkDffMcQZfe7SOioeJjnnR5FHHwBdnQ9VXcMINcZ8duNemFIQF7Pq++irsJviB2gmjwF5bO9Z5aUsgsSAIqYOIF4u4PV7OuX8pNz2zsqO3knY0u71JN9eX+C0vuZnOsPGqBldSn9MWeudGW15yM1vElturo9KCM4IxMS1uozscC3BWbIWLH4mdZRS8U/mfG255CeiYlGrMaDAmlhdBEOIh4sUitX6Ly/Idh4NjlXXNfBzyXoimqsHFmF+9zaNLdyV13UDciDPCdNHo8iT1OW0hEPMSaklwhiiZP7+zlT3l4Q0lAzEvvXa8zH3Oh7j40N/5puNdaouuhmOvNo5wNcGuVHjMi1/J6OD/dTxSpE4QhNYgAbsWqWmMdhdd9+8vWLevim13zw75F7MQSnmtz73z3y++4uYZI5O27uHaZgBqI/5c6ps7Vrw4bCpYdG7CgB5A+GEc+nrzweqo+zMcNqjYzeBP76SPzU1OZRPVOpuak36EtbJvLdhsKsziFXilUyjmxahInbh2BEGIh5y4FjESLxv2VwHw9oaDR3s7aUNATCQ7xiJgYalOIfHi9njDquWePMLXNDHUDTKisFvMNRyeBnjqQpR2c0nzb7l9xJuc2PQQ3tx+Ce8n0oIRtLykhm4RBEFoNSJeYrB27xEue/gTdpXVUtMYHUsROKd+8PwaVuypOMq7Sw+q/TEoyf6Xfot4Cf9zaWileNl4oIp9lfXxJ8YgUK/l1pkj+eznZwbFQ6jb6HuzRvHvb57AF78803ANteRPULWX3ec+yVY9hAaVRSOZiXiLwgJ2w0JeQmNeUkTAiJFFEITWIOIlBp/vLmdlcSVn3LeEN9b5rCtmv2xLIzoGCz4C4iLplhe3T6Q0u8PbAtS3MublggeWM+1PH7VpTwHx0qd7Jv16ZAXHAyJGKXDYbcwa24c+3bMYGWGFmaK2wScPwpTrqB9wGhCSKp3AKR+YabdF9jZqiXlJEe3SLgzI9xXwu/z4QR28E0EQ2guJeYlBWYggWba9LObcyENU8FHd4HPrJN9tZPz9fn3tAWaNLeTSKdYPriZ3tODZfbiOj7aUcv2pwywXTQusE1kkL3B/aLAuQF62E9CMUAf5lv1t5to/gR6D4Zy7UeUt1XChdRaKyIDdwOv27G2UKO1heSnolsHOP56PxP0KQuelQ8SLUmoecBcwHjhRa70i5NrPgRsAD3Cb1vpd//h5wN8AO/BPrfX89t5nqHiprPdZEMx+HxodgEK0WydZxHIP/fDFtQmJl/vf3x627hn3LQ5mM504vICJA3vEXeOdDQe5+dlVAGQ6w0VKIObFGThNt7wJG1/h/so1FGZ+RTfVRJN28oH3OM6/7p+QlYdSvniqQP8jo8BWMwIBr5EBu4GYF1IoVbq9kIwlQejcdJTlZQNwKfBo6KBSqgi4CpgADAAWKaXG+C//Azgb2Ad8qZT6n9Z6U3tsrrmpkV3rPubg3iOAz/wfr36IkSWgoq6Zg1UNwayTUF5fe4DcTAezxvUJjq3Ze4SVxZVcdcJgumU60FqzfMdhTh3ZO21/GQdiXkKbD5bWNFKYmxmWVaK1xuXRUVlbH+84TI9sZ5SAaGyDWNx/pIGeOU5cbk1ZbSOPLNkZvPbSyr1B4QJQVttEVb2LPeV1wVYEO0pr2X+kgRljCoPz3lxfEnwdKDQXwOFtZrA6xMn2vfDs47BjEXQrpME5lBcbZlI4bBJ/2jmQvbove3oOBVrESlRjxgRwRFbYDca8pE7QbiKiTBAEIUCHiBet9WYwTImcCzyvtW4CdiuldgAn+q/t0Frv8t/3vH9uu4iXmopSxr15KRNd3+CImsgRncshevIzx/P0pIaNT3/ApsxjKcDJMFXCQd2L6voWK83HOw7zxPLdfLilFIDd95wf9Vm//9/VwWsLV+1nytCe/Oa1DazdV0W3DDtXnTiEpdsPc92/vuD7Z4zik53lXDi5P9efNrxVn0lrzUOLd1Je28zqvZU88o3j6ZuXFf9GP40uD48t3cX1pw0jL8sZdf2fy3aRl+3kFwvX8+g1x3Pm+L68uno/D3y4A/BZYDbsr+JIvYtvPPE5s8YWcurI3pxd1JcfvriGVV8dIT/HyTs/OJ1+PbJwe7zUNXv4+j8/B2DP/AvCntdk4jYCyHKah3I1NHs4bf6HzBpbyI6yWvZWNIRd/7t/vwEO1zTxnWdW8MXuCrb8/jye/ayYu9/cDMC2u2dTVttE3+6ZrAwJ2A42X/R6Yd0LHP/unSzLLPX5bQ71h5m/gGm387t/rebTinL+dMwknr64V1hQeOCvS9BtZPqJognM7Z7lpCakIrQ3JGA3VZCAXUEQWkOqxbwMBD4Leb/PPwawN2L8JKMFlFI3AjcCDBkypFWbyCscSLXO5tfOZ4Nj+3UvBqpyynV3eu5cwgSlmRdy9lcvy8ZVcT4b+1zEd97x4MJBITU48VBWeYQ+BT2Dc+ubWw6UJ5bv5u43NzMwPzv4i/xnC9eT4bCx1d+n50H/gbqyuJK8bCe9cjM5eUQB+yobyM108N6mQ5w+ujcOu42B/mDFtXuP8Me3NnPF1MFcctxA/rpoW1BIACxYuY/vzhoF+Hrs2GwKr1ez9VANWsP8d7bQ5PLwx0sn0dDsYcHKfTz5yR4cdsWtM0cF11n9VSWPL9vFWyGWh1+9uoEzxvXhnrc3h3xmD3MeXB58/9HWMj7aWsZbGw6y+qsjABypd/HY0l0cNySfdzeWBIOkQ7n//W0s2nyIg1UNUdcCxGrM+NHW0uDzQ/nf907jxy+tZduh2rDxwzVNfLmnHFAUl9fzckgTxac+3sWf3t7I5MG9OFDVSAYuCqhm4L634cBXsO1dOLSB5t6Tubt6NqU5Y3jk9pvB7vuxK/PXwJk6rIDhvcODd0PbA/gGTD+SKT27ZbC3oiWDKiCENh6Iri/TUcT7WBMH5rFhf+rsVxCE1KDdxItSahFgVJzil1rr18xuMxjTGGdFGf7zUWv9GPAYwNSpU1v1T0ynw85mBjAZnzvBoxXdaWBd1lQG3vo6x9/7PnO8HzJYlTFK7WedHkEfKrliy5scu+llNkUaNB64DZ2RS52jJzk9++F29OQvzkZyaSD7vSaecXqx13mxKy+2DC92vNhe9TIaLxdlaGz4xgLjdrzUZdrIaXbhRjFdO3Bjw4PmoNMG2ktPt4e/AOqA5tDrcKXWXJWpUYBC41iqaPzMRiOZVHi7UdvsRSsbpZ7u1JHJXGxk4mLNAxn8//buPDyq6m7g+Pc3M5nMZN8IYEjYMUEkLALugqCAtC5Vi1atbX3sq7V16Wtd6+tWnr5tbW1t+1Zbq8W2VlstWqUtWKUW+7AIFRGFtAooLqwJW8g2M7/3j3snzJIEEiDJDL/P8+SZuefeuZzfZMj9zTnnntOsPgbj44GMJo5ZvI+WDwqob4GQ+GjeVs+svU2ckyEIip8QuQ0N1P/Yyy8a6yktDuDXJprr69isRTTinK+JDJrw07Alh1m+DErzs9jbHKFuWZiNyzwMUR/Xen204KUFH7xRB9l9WPzKO/jwUIWXQZ4t7CKbflLLYPmEStmElwhCJjzzFPiC7GgScrODCMqKDTvYsWU3c3yK4Px4UIYV+xm9+Lc8Hazjff8WBuUqQd1HY/0ecl5t4NrMCE3qg0d8zIuE8QUieDSCZ5FydQDCW4WP/CWUyk4C0gL/BMQDZePh/J/x78LpPPGzpVRkZLUmLgDfv7ia51d9zJCS5LlfoncXRbvbutK9UhDMYE1Md2dLOMIx+QF2N4aYPaGcX762odPnPNwONCHdc185pXXcjzHGRB2x5EVVp3XhZR8C5THbA4DosrvtlR8RIfWAwFOhycwJXc4egvzXpKHcnpdF9ZD+/Lrm7KTXbJ14K2uXLmCs5z8UFhQxuKKCZ9/cQjG7KQntolh2M84fIi+ykRM9tRQXFVOzI0QYD2E8RPCg3kya8TC4NI/djRFK8oIs37iT8uIcCvKyWLJhJ4iHlkYPYfXgIYJPwvgIowiRZueynBvIoDQvwJ7GMBk+L/lBP8U5mWzb20yGz8PrG3eiLZAjThLlJULQpwz0bsMfaSIv00Ntk5Av9ShCQSZsa/JS25LLO+/WkkEYHyEKgf4ZHrIyvdTVt9CMjz1ksXybDyGf4SXFkBHgb+v2Uibb8UmYHGmgmN0EaCaPenL9LQSaheaWFvC6iZokXLCenwvAHzPb/53VRAbQgJ9c3Un9hi1EWhqhaR+NhBDxUKnCCK8Q9GfQGFLE4yE/KxOP+KAul8LMHAKDygnk5CP+XJ5csY16AoTw4idERjhEGC9nVPbllX/X0hTyEEHIkQb6SS3bI/ls0lIuufAiqkZPAp+7MOMmp2UpcSmD6vKC1nE0iaJHLl3vdEd1asiTe2xe0EdtfTOTv7eIe849jo92NnDZpIHcPrMSn9fTK5KXA/F5Pb2uedgY0/N629+FPwFPisgPcAbsDgeW4/w5Hi4ig4GPcAb1fu5IVmRueDo+QtwfuoJ6nK6YMe6FJjqPRKKlm2FpZAILIhO4edwILpo8jHs+eY13YqeBd3tX/F4Pb391OgMaWnh08YbWQaMPXFzNhePKEBH6ui8ZtbOB/nkBPB5h0q5G6vY1M/NHi1tPedmkCm6dWUkk4gx4XbVpJxMGFSXdmgtQAuxpbOHVP6xmU92+1i6EtffNIOj3EokoH9Y1UFScxR9efY9v/2UdJTl+VtxxFk+/9G8eevk/SeecOqSUyZWl3PXcGiYOLuLskX15YGENX5k8jClTh6Oq3P0/C2hoZw6WK08ayL3njeJLjy7jtXe3M+v4/gwrCfDwohoyCJFBiBeurmbJWzW8uHwtHiIEaOYDLSVP9rFbs1mrFZwwqITlG2vJyfSxd0fyjMgXjx/AfeeNIuj3khlRPJL8zT/2N/vdlX+Nq3Nxtp/jyvKZfcEYrs708f2FNQwrzeEbz6wGnLEuTeEI03KrWhMX2H/nS1u/j/YkNkh0Zcr8/KAzNmnjjn184fHXARhUnIWvE/U40mzIizGmK3rqVukLgB8DfYD5IrJKVaer6tsi8nucgbgh4DpVDbuv+SqwAOdW6cdU9e0jWcfnwyfzfPjk1u0FN57OiL45APRrY6DrwOIsVrnfsMEZLOn1CH++4TTAWePn8X9u5CeLnHEnzeEIGV4PJTmZ3DazkomDC/nSr1Zw3DF5SReqsphkqV9+gL55mVw7eSi5AR8PLKhh9oTyuEG0Jw8t6TC23EAGD18xnl37Wqi+byEAQb8zTsTjESqKswAYUOg8toSdVpCvnzWCXfuambvk/bjzNYbCTKsqZdG6UuZcMIr++UGuOnVwaxwiQm7A15oIXH/mMIqy/dzzgjPeOj/LWcAwL+h8HPvkZnLT9JG8+fFe/u6OTTn3yY+orfcD1fHBxDTQjB6Qz+gB+TzaTotCdXlBa5wHc/fWa7dOYfy3/gbA/eeP4ooTB8btv/2cKoDW5OXhK8Yz718fMWFQUdxx0a6fziQviZf1zlzk75o1kjufe4v++clJ9sDi/V1UC286nYJg8uDr7mQDdo0xXdFTdxvNA+a1s28OMKeN8j8Dfz7CVWvXsf32L4uXF3Detk9XH8MLbzq9V/3yArwfs0JwbiD+rS3OyaSiKKt1e2rMLdIAZ1b2Zd39MzocbBolItw6oxKA2SeUU5zTQV9KB/KzMhhVlsf0kW2vmzOg0Ln4hcL77+y5c9ZIrpsyjLxgBjc9vYq/rNlMcXYm/fODPPaFCXF1jBWN68Zpw7lx2gg2bK+HaPLiXkCjxxRnO8nMw5ePZ/YjS1i3eQ+19c3txpHt91LfHGZqVV9eWbcFgCtOHEjNlj0s37D/LqDThnec1CWKfV9PGFjYwZGO48vymXJsaVJ5KOK8f4ndRh3JTLhlvDMX+c9OKOezE8r5y1vJA55j11Ya0bezSz0eft29CGP0/3FbiZ0xJnX0tm6jXiM34GNPY4j//czxcVO9Q9t3s5QVBiHmC39uG7cTR1s0AB66dGzS/oNJXBJ1NXGJevFrp7W7L5q8RFsrwFn1uNRtefq/y8bx1OubmDnqwIsGRls6ojHmZO7/6EWTl2gLRaGbvAQyvEyt6subH+6KO9eQkmzWb6/H7/PQHIpwwqAi5n7JuaP+5/9wut9OGlrMnbOq+MLjy1m6vpaV35zWpfeqKNtPbX0zxx7EhT6/nVaMaMtVZ1peSvPi69qVAbvRRqlxFQX86JKx7G5saW1NO1p9+bQhTBpcxNiKAyejxpjey5KXdsz7ysksWV/LJROTb7ee5K4WfOG4staWl8p+8Re3vEDyW1vutryMqyggO7P3v/VF2X6umzKUmaP6t7lfRLi0jfenLdFemoDbohDbMhW96Ifci3zsvix/fEJ396dHsnF7Peu317cuyRA7r8sN00awuzHEGSP6EMjw8ssrJ1CzZU+Xk7z515/Kzn0tB7VEQHvJSbQVpTOJQ+JEd10ZHDJhUBH98gLce+6o1s/e0c7jEUtcjEkDvf8K2kOGleYyrLTtb9uDS7KTJk07tl9e3HZbyUlZQZBvnT+KaVV9k/b1RiLCN6ZXHpZzJba8xHaLFGQ5yUtLONq9sn9f7Pt4wdgyvnjKYHY1tFC3r4WwKvNXfxLXYjWmvIBnrz057vXjDuFi1T8/eMAuhjvPqeKf721vd//YikJ+OHsMZx/X9d97V3pX+uRmsvSOtlev7o3KCoJHbDkJY0x6seTlMKnqH5/otHexuTxh0OfRIjpvSbQLKnasQ3SCtlFl+Sx8Z0vcAOXYKT4a3QG/+cEMHrp0LN/96zrnnF3objucrj59CFefPqTDY84fW9bh/gPpzKrSqWrxLVN6ugrGmBRhycsheuDiair75VKaG+CNu86ivjnEk8s+oCqhJeZoF01WkrpDgBK3S+e6KcM4Y0SfuLlP6vbtH6h728z4VqDoelNl7dy6nuoq++WybrMzy3JvWQX6SLhtZiWnD+8T1zX39bNGMHrAgRfENMYcnSx5OUQXjd+/enFhtp/CbD+3zDg8XS3ppHXMS8z4lGevPYmi7Pj5UBInbTtjRB++t6CGF792atxtvgAXjh9AKKwHbPVIVc9cezI1m/dQs3lPmwPA08U1ZwxNKrt+6vAeqIkxJlVY8mK6RbTbI3Z8yviBRe0d3mpUWX7S+KKocRWFhzSepbfLyfQxfmAh4w/iNm1jjDma9J6pNk1ai7a8dG6iNmOMMSaZXUlMt9g/niF9x24YY4zpHpa8mG6RkwLz2hhjjEkNdkUx3eL7F1fzxJL3GVtu4zeMMcYcGkteTLcozQtw8/Rje7oaxhhj0oB1GxljjDEmpVjyYowxxpiUYsmLMcYYY1KKJS/GGGOMSSmWvBhjjDEmpVjyYowxxpiUYsmLMcYYY1KKJS/GGGOMSSmimr5rzYjINuD9Lr68BNh+GKvTG6V7jOkeH6R/jBZfzxioqn16uhLGtCetk5dDISIrVPWEnq7HkZTuMaZ7fJD+MVp8xpi2WLeRMcYYY1KKJS/GGGOMSSmWvLTv5z1dgW6Q7jGme3yQ/jFafMaYJDbmxRhjjDEpxVpejDHGGJNSLHkxxhhjTEqx5KUNIjJDRGpE5F0Rua2n69MVIvKYiGwVkTUxZUUi8pKI/Md9LHTLRUQecuNdLSLjeq7mB0dEykVkkYisFZG3ReQGtzydYgyIyHIRedON8V63fLCILHNjfFpE/G55prv9rrt/UE/W/2CJiFdE3hCRF93tdItvo4i8JSKrRGSFW5Y2n1NjeoIlLwlExAv8FJgJjAQuFZGRPVurLvkVMCOh7DbgZVUdDrzsboMT63D358vAz7qpjociBPy3qlYBJwLXub+ndIqxCThTVauBMcAMETkR+A7woBtjHXCVe/xVQJ2qDgMedI9LBTcAa2O20y0+gCmqOiZmTpd0+pwa0+0seUk2EXhXVderajPwFHBeD9ep01T1H0BtQvF5wFz3+Vzg/JjyJ9SxFCgQkf7dU9OuUdVPVPVf7vM9OBe/MtIrRlXVve5mhvujwJnAM255YozR2J8BpoqIdFN1u0REBgCzgEfdbSGN4utA2nxOjekJlrwkKwM2xWx/6Jalg76q+gk4F3+g1C1P6Zjd7oOxwDLSLEa3S2UVsBV4CXgP2KmqIfeQ2DhaY3T37wKKu7fGnfZD4BYg4m4Xk17xgZNwLhSRlSLyZbcsrT6nxnQ3X09XoBdq65tcut9PnrIxi0gO8Cxwo6ru7uCLeErGqKphYIyIFADzgKq2DnMfUypGEfkUsFVVV4rI5GhxG4emZHwxTlHVj0WkFHhJRNZ1cGyqxmhMt7KWl2QfAuUx2wOAj3uoLofblmgTtPu41S1PyZhFJAMncfmtqv7RLU6rGKNUdSfwd5zxPQUiEv3iERtHa4zu/nySuw57k1OAc0VkI0737Jk4LTHpEh8Aqvqx+7gVJwGdSJp+To3pLpa8JHsdGO7e8eAHLgH+1MN1Olz+BFzpPr8SeD6m/PPunQ4nAruiTdq9lTvW4ZfAWlX9QcyudIqxj9vigogEgWk4Y3sWARe5hyXGGI39IuAV7cWzUKrq7ao6QFUH4fw/e0VVLyNN4gMQkWwRyY0+B84G1pBGn1NjeoLNsNsGETkH5xugF3hMVef0cJU6TUR+B0wGSoAtwN3Ac8DvgQrgA+BiVa11E4Gf4NydtA/4oqqu6Il6HywRORVYDLzF/vESd+CMe0mXGEfjDOb04nzR+L2q3iciQ3BaKoqAN4DLVbVJRALAr3HG/9QCl6jq+p6pfee43UY3q+qn0ik+N5Z57qYFkgYQAAADDElEQVQPeFJV54hIMWnyOTWmJ1jyYowxxpiUYt1GxhhjjEkplrwYY4wxJqVY8mKMMcaYlGLJizHGGGNSiiUvxhhjjEkplrwYE0NEwu7qv9GfDlcVF5FrROTzh+Hf3SgiJYd6Hvdc2SLykvv8tZgJ34wxJi3YHzVj4jWo6piDPVhVHz6Slemik4ClIlII1MesE2SMMWnBWl6MOQhuy8h3RGS5+zPMLb9HRG52n18vIu+IyGoRecotKxKR59yype7Ec4hIsYgsFJE3ROQRYta0EZHL3X9jlYg84i7O6BWRX4nIGhF5S0RuaqOOQ91FHH8DfA5YCVS75ylNPN4YY1KVJS/GxAsmdBvNjtm3W1Un4syA+sM2XnsbMFZVRwPXuGX3Am+4ZXcAT7jldwOvqepYnCnhKwBEpAqYjbOY3xggDFwGjAHKVHWUqh4PPJ74j6vqe+5rVuKsn/MEcJWqjnHX1THGmLRg3UbGxOuo2+h3MY8PtrF/NfBbEXkOZykGgFOBCwFU9RW3xSUfOB34jFs+X0Tq3OOnAuOB190VsoM4i/a9AAwRkR8D84GFHcRQqqo7ROR44BcHCtgYY1KNtbwYc/C0nedRs4Cf4iQfK92BstLGcZrwGEuAuW5ryRhVPVZV71HVOqAaZ2Xp64BHk14o8rCIrMFZWHQVzvo489vqYjLGmFRmyYsxB292zOOS2B0i4gHKVXURcAtQAOQA/8Dp9okuPrhdVXcnlM8ECt1TvQxcFB2j4o6ZGejeieRR1WeBu4BxiZVT1WtwuqnuB84H5rsJUFutRMYYk7Ks28iYeEG31SLqr6oavV06U0SW4ST9lya8zgv8xu0SEuBBVd0pIvcAj4vIapxVgq90j78X+J2I/At4FWdlYVT1HRH5JrDQTYhacFpaGtzzRL9w3N5O/c/AGetymnteY4xJO7aqtDEHQUQ2Aieo6vaerosxxhztrNvIGGOMMSnFWl6MMcYYk1Ks5cUYY4wxKcWSF2OMMcakFEtejDHGGJNSLHkxxhhjTEqx5MUYY4wxKeX/AYfs/ZNQI1iXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print('length of scores: ', len(scores), ', len of avg_scores: ', len(avg_scores))\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores, label=\"Score\")\n",
    "plt.plot(np.arange(1, len(avg_scores)+1), avg_scores, label=\"Avg on 100 episodes\")\n",
    "plt.legend(bbox_to_anchor=(1.05, 1)) \n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episodes #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play(env, agent, num_episodes):\n",
    "    \n",
    "    state = env.reset()\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    \n",
    "    for i_episode in range(num_episodes + 1):\n",
    "        \n",
    "        state = env.reset()\n",
    "        score = 0                    \n",
    "        time_start = time.time()\n",
    "        \n",
    "        while True:\n",
    "            \n",
    "            action = agent.select_action(state, eval=True)\n",
    "            env.render()\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            score += reward \n",
    "            state = next_state\n",
    "    \n",
    "            if done:\n",
    "                break\n",
    "                \n",
    "        s = (int)(time.time() - time_start)\n",
    "        \n",
    "        scores_deque.append(score)\n",
    "        scores.append(score)    \n",
    "        \n",
    "        print('Episode {}\\tAverage Score: {:.2f},\\tScore: {:.2f} \\tTime: {:02}:{:02}:{:02}'\\\n",
    "                  .format(i_episode, np.mean(scores_deque), score, s//3600, s%3600//60, s%60)) \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 0\tAverage Score: 318.53,\tScore: 318.53 \tTime: 00:00:16\n",
      "Episode 1\tAverage Score: 318.56,\tScore: 318.59 \tTime: 00:00:13\n",
      "Episode 2\tAverage Score: 318.68,\tScore: 318.92 \tTime: 00:00:14\n",
      "Episode 3\tAverage Score: 318.59,\tScore: 318.35 \tTime: 00:00:14\n",
      "Episode 4\tAverage Score: 318.50,\tScore: 318.10 \tTime: 00:00:14\n",
      "Episode 5\tAverage Score: 318.36,\tScore: 317.66 \tTime: 00:00:14\n"
     ]
    }
   ],
   "source": [
    "play(env=env, agent=agent, num_episodes=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kernel-ml",
   "language": "python",
   "name": "kernel-ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
